436 lines
18 KiB
Python
436 lines
18 KiB
Python
import os
|
|
import sys
|
|
import time
|
|
import hashlib
|
|
import subprocess
|
|
import threading
|
|
from pathlib import Path
|
|
from typing import Optional, List, Dict, Tuple
|
|
import psutil
|
|
|
|
from core.models import TaskConfig, TaskStatus, TaskMetrics
|
|
from core.python_env import PythonEnvManager
|
|
from core.logger import log_info, log_warn, log_err, log_setup, log_debug
|
|
|
|
|
|
class ManagedTask:
|
|
"""Управление жизненным циклом отдельной задачи/процесса"""
|
|
|
|
def __init__(self, config: TaskConfig, logs_dir: str = "logs"):
|
|
self.cfg = config
|
|
self.logs_dir = Path(logs_dir)
|
|
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
self.stdout_path = self.logs_dir / f"{self.cfg.name}_stdout.log"
|
|
self.stderr_path = self.logs_dir / f"{self.cfg.name}_stderr.log"
|
|
|
|
self._stdout_file = None
|
|
self._stderr_file = None
|
|
|
|
self.proc: Optional[subprocess.Popen] = None
|
|
self.psutil_proc: Optional[psutil.Process] = None
|
|
|
|
self.status: TaskStatus = TaskStatus.STOPPED
|
|
self.start_time: float = 0.0
|
|
self.stop_time: float = 0.0
|
|
self.restart_count: int = 0
|
|
self.last_crash_time: float = 0.0
|
|
self.last_exit_code: Optional[int] = None
|
|
self.last_error_message: str = ""
|
|
|
|
# Планировщик (Interval / Cron)
|
|
self.next_run_time: float = 0.0
|
|
|
|
self._lock = threading.RLock()
|
|
self._installed_reqs_hash: Optional[str] = None
|
|
|
|
def update_config(self, new_cfg: TaskConfig):
|
|
"""Обновление конфигурации задачи на лету"""
|
|
with self._lock:
|
|
self.cfg = new_cfg
|
|
|
|
def get_python_executable(self) -> Tuple[str, Optional[str]]:
|
|
"""
|
|
Определение интерпретатора Python с учетом auto-create venv,
|
|
выбора версии Python (Вариант А + Резерв Б) или python_path.
|
|
Возвращает: (путь_к_python, путь_к_папке_bin_для_PATH)
|
|
"""
|
|
# 1. Если указан прямой python_path
|
|
if self.cfg.python_path and os.path.exists(self.cfg.python_path):
|
|
return self.cfg.python_path, None
|
|
|
|
# 2. Использование PythonEnvManager (автосоздание venv, поиск версии, докачка)
|
|
req_path = self.cfg.requirements_path if self.cfg.requirements_enabled else None
|
|
return PythonEnvManager.ensure_task_environment(
|
|
venv_path_str=self.cfg.venv,
|
|
python_version=self.cfg.python_version,
|
|
auto_create_venv=self.cfg.auto_create_venv,
|
|
allow_download=self.cfg.allow_download_python,
|
|
requirements_path=req_path,
|
|
requirements_enabled=self.cfg.requirements_enabled
|
|
)
|
|
|
|
def check_and_install_requirements(self, force: bool = False) -> bool:
|
|
"""Проверка и установка зависимостей задачи с кэшированием хеша файла"""
|
|
if not self.cfg.requirements_enabled and not force:
|
|
return True
|
|
|
|
req_path = self.cfg.requirements_path
|
|
if not req_path:
|
|
return True
|
|
|
|
candidate_paths = [
|
|
Path(req_path),
|
|
Path(self.cfg.cwd or ".") / req_path,
|
|
Path(os.path.dirname(os.path.abspath(self.cfg.script))) / os.path.basename(req_path)
|
|
]
|
|
target_req = None
|
|
for p in candidate_paths:
|
|
if p.exists() and p.is_file():
|
|
target_req = p
|
|
break
|
|
|
|
if not target_req:
|
|
log_warn(f"[{self.cfg.name}] Файл зависимостей не найден: {req_path}")
|
|
return False
|
|
|
|
try:
|
|
with open(target_req, "rb") as f:
|
|
content_hash = hashlib.md5(f.read()).hexdigest()
|
|
except Exception as e:
|
|
log_err(f"[{self.cfg.name}] Не удалось прочитать requirements: {e}")
|
|
return False
|
|
|
|
if not force and self._installed_reqs_hash == content_hash:
|
|
log_debug(f"[{self.cfg.name}] Зависимости уже актуальны (хэш совпадает)")
|
|
return True
|
|
|
|
python_bin, _ = self.get_python_executable()
|
|
log_setup(f"[{self.cfg.name}] Установка зависимостей из {target_req} через {python_bin}...")
|
|
try:
|
|
result = subprocess.run(
|
|
[python_bin, "-m", "pip", "install", "-r", str(target_req)],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
timeout=180
|
|
)
|
|
if result.returncode == 0:
|
|
self._installed_reqs_hash = content_hash
|
|
log_setup(f"[{self.cfg.name}] Зависимости успешно установлены!")
|
|
return True
|
|
else:
|
|
log_err(f"[{self.cfg.name}] Ошибка pip: {result.stderr.strip()}")
|
|
return False
|
|
except subprocess.TimeoutExpired:
|
|
log_err(f"[{self.cfg.name}] Таймаут установки зависимостей (180 сек)")
|
|
return False
|
|
except Exception as e:
|
|
log_err(f"[{self.cfg.name}] Ошибка вызова pip: {e}")
|
|
return False
|
|
|
|
def rotate_logs_if_needed(self, max_size_bytes: int, backup_count: int = 3):
|
|
"""Ротация файлов логов при превышении максимального размера"""
|
|
with self._lock:
|
|
for log_path in [self.stdout_path, self.stderr_path]:
|
|
if not log_path.exists():
|
|
continue
|
|
|
|
try:
|
|
if log_path.stat().st_size >= max_size_bytes:
|
|
self._close_log_handles()
|
|
|
|
# Ротация существующих бэкапов: .log.3 удаляется, .log.2 -> .3, .log.1 -> .2, .log -> .1
|
|
for i in range(backup_count, 0, -1):
|
|
src = Path(f"{log_path}.{i}")
|
|
dst = Path(f"{log_path}.{i + 1}")
|
|
if i == backup_count and src.exists():
|
|
src.unlink(missing_ok=True)
|
|
elif src.exists():
|
|
src.rename(dst)
|
|
|
|
if log_path.exists():
|
|
log_path.rename(Path(f"{log_path}.1"))
|
|
|
|
log_debug(f"[{self.cfg.name}] Выполнена ротация лога {log_path.name}")
|
|
self._reopen_log_handles()
|
|
except Exception as e:
|
|
log_debug(f"Ошибка ротации лога {log_path}: {e}")
|
|
|
|
def _reopen_log_handles(self):
|
|
"""Переоткрытие дескрипторов после ротации"""
|
|
if self.is_running():
|
|
try:
|
|
self._stdout_file = open(self.stdout_path, "a", encoding="utf-8", buffering=1)
|
|
self._stderr_file = open(self.stderr_path, "a", encoding="utf-8", buffering=1)
|
|
except Exception:
|
|
pass
|
|
|
|
def start(self) -> bool:
|
|
"""Запуск процесса задачи"""
|
|
with self._lock:
|
|
if self.is_running():
|
|
log_warn(f"Задача {self.cfg.name} уже выполняется (PID: {self.proc.pid})")
|
|
return False
|
|
|
|
script_path = Path(self.cfg.script)
|
|
if not script_path.exists():
|
|
if self.cfg.cwd:
|
|
script_path = Path(self.cfg.cwd) / self.cfg.script
|
|
if not script_path.exists():
|
|
log_err(f"[{self.cfg.name}] Скрипт не найден: {self.cfg.script}")
|
|
self.status = TaskStatus.CRASHED
|
|
self.last_error_message = f"Файл не найден: {self.cfg.script}"
|
|
return False
|
|
|
|
script_abs = script_path.resolve()
|
|
working_dir = Path(self.cfg.cwd).resolve() if self.cfg.cwd else script_abs.parent
|
|
|
|
# Установка зависимостей
|
|
if self.cfg.requirements_enabled:
|
|
self.check_and_install_requirements()
|
|
|
|
# Определение интерпретатора Python
|
|
python_bin, bin_dir = self.get_python_executable()
|
|
|
|
# Подготовка окружения
|
|
env = os.environ.copy()
|
|
env["PYTHONUNBUFFERED"] = "1"
|
|
if bin_dir:
|
|
env["PATH"] = bin_dir + os.pathsep + env.get("PATH", "")
|
|
env["VIRTUAL_ENV"] = str(Path(bin_dir).parent)
|
|
|
|
if self.cfg.env:
|
|
env.update({k: str(v) for k, v in self.cfg.env.items()})
|
|
|
|
cmd = [python_bin, str(script_abs)] + list(self.cfg.args)
|
|
|
|
try:
|
|
self._close_log_handles()
|
|
self._stdout_file = open(self.stdout_path, "a", encoding="utf-8", buffering=1)
|
|
self._stderr_file = open(self.stderr_path, "a", encoding="utf-8", buffering=1)
|
|
|
|
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
banner = f"\n{'='*20} [START {timestamp}] {'='*20}\n"
|
|
self._stdout_file.write(banner)
|
|
self._stderr_file.write(banner)
|
|
self._stdout_file.flush()
|
|
self._stderr_file.flush()
|
|
|
|
self.proc = subprocess.Popen(
|
|
cmd,
|
|
stdout=self._stdout_file,
|
|
stderr=self._stderr_file,
|
|
cwd=str(working_dir),
|
|
env=env
|
|
)
|
|
|
|
self.start_time = time.time()
|
|
self.status = TaskStatus.RUNNING
|
|
try:
|
|
self.psutil_proc = psutil.Process(self.proc.pid)
|
|
self.psutil_proc.cpu_percent(interval=None)
|
|
except Exception:
|
|
self.psutil_proc = None
|
|
|
|
log_info(f"Задача '{self.cfg.name}' запущена (PID {self.proc.pid})")
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.status = TaskStatus.CRASHED
|
|
self.last_error_message = str(e)
|
|
log_err(f"Ошибка запуска задачи '{self.cfg.name}': {e}")
|
|
self._close_log_handles()
|
|
return False
|
|
|
|
def stop(self, force: bool = False, timeout: float = 5.0) -> bool:
|
|
"""Остановка процесса и его дочерних процессов"""
|
|
with self._lock:
|
|
if not self.proc or self.proc.poll() is not None:
|
|
self.status = TaskStatus.STOPPED
|
|
self._close_log_handles()
|
|
return True
|
|
|
|
self.status = TaskStatus.STOPPING
|
|
pid = self.proc.pid
|
|
|
|
children = []
|
|
try:
|
|
if self.psutil_proc and self.psutil_proc.is_running():
|
|
children = self.psutil_proc.children(recursive=True)
|
|
except Exception:
|
|
pass
|
|
|
|
log_info(f"Остановка задачи '{self.cfg.name}' (PID: {pid})...")
|
|
|
|
try:
|
|
if force:
|
|
self._kill_process_tree(children)
|
|
else:
|
|
self.proc.terminate()
|
|
for child in children:
|
|
try:
|
|
child.terminate()
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
self.proc.wait(timeout=timeout)
|
|
except subprocess.TimeoutExpired:
|
|
log_warn(f"Задача '{self.cfg.name}' не ответила на SIGTERM, принудительное завершение...")
|
|
self._kill_process_tree(children)
|
|
|
|
except Exception as e:
|
|
log_err(f"Исключение при остановке '{self.cfg.name}': {e}")
|
|
finally:
|
|
self.stop_time = time.time()
|
|
self.status = TaskStatus.STOPPED
|
|
self._close_log_handles()
|
|
|
|
log_info(f"Задача '{self.cfg.name}' остановлена")
|
|
return True
|
|
|
|
def _kill_process_tree(self, children: List[psutil.Process]):
|
|
for child in children:
|
|
try:
|
|
child.kill()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
if self.proc:
|
|
self.proc.kill()
|
|
self.proc.wait(timeout=2.0)
|
|
except Exception:
|
|
pass
|
|
|
|
def _close_log_handles(self):
|
|
"""Безопасное закрытие дескрипторов файлов логов"""
|
|
if self._stdout_file:
|
|
try:
|
|
self._stdout_file.flush()
|
|
self._stdout_file.close()
|
|
except Exception:
|
|
pass
|
|
self._stdout_file = None
|
|
|
|
if self._stderr_file:
|
|
try:
|
|
self._stderr_file.flush()
|
|
self._stderr_file.close()
|
|
except Exception:
|
|
pass
|
|
self._stderr_file = None
|
|
|
|
def is_running(self) -> bool:
|
|
if self.proc is None:
|
|
return False
|
|
return self.proc.poll() is None
|
|
|
|
def poll_exit(self) -> Optional[int]:
|
|
with self._lock:
|
|
if self.proc is None:
|
|
return None
|
|
exit_code = self.proc.poll()
|
|
if exit_code is not None:
|
|
self._close_log_handles()
|
|
self.last_exit_code = exit_code
|
|
self.stop_time = time.time()
|
|
self.psutil_proc = None
|
|
return exit_code
|
|
|
|
def check_stable_run(self):
|
|
with self._lock:
|
|
if self.is_running():
|
|
uptime = time.time() - self.start_time
|
|
if uptime > self.cfg.stable_uptime and self.restart_count > 0:
|
|
log_debug(f"[{self.cfg.name}] Стабильная работа ({uptime:.0f}с), сброс счетчика рестартов")
|
|
self.restart_count = 0
|
|
|
|
def check_memory_limit(self) -> Optional[float]:
|
|
"""Проверяет превышение лимита памяти. Возвращает текущий объем RAM (MB) при превышении, иначе None"""
|
|
if not self.is_running() or not self.cfg.max_memory_mb:
|
|
return None
|
|
|
|
try:
|
|
if self.psutil_proc and self.psutil_proc.is_running():
|
|
mem_mb = self.psutil_proc.memory_info().rss / (1024 * 1024)
|
|
if mem_mb > self.cfg.max_memory_mb:
|
|
return round(mem_mb, 1)
|
|
except Exception:
|
|
pass
|
|
return None
|
|
|
|
def collect_metrics(self) -> TaskMetrics:
|
|
with self._lock:
|
|
if not self.is_running():
|
|
return TaskMetrics(
|
|
status=self.status,
|
|
restart_count=self.restart_count,
|
|
last_exit_code=self.last_exit_code,
|
|
last_error=self.last_error_message
|
|
)
|
|
|
|
uptime = time.time() - self.start_time
|
|
cpu = 0.0
|
|
mem_mb = 0.0
|
|
|
|
try:
|
|
if self.psutil_proc and self.psutil_proc.is_running():
|
|
cpu = self.psutil_proc.cpu_percent(interval=None)
|
|
mem_mb = self.psutil_proc.memory_info().rss / (1024 * 1024)
|
|
else:
|
|
self.psutil_proc = psutil.Process(self.proc.pid)
|
|
cpu = self.psutil_proc.cpu_percent(interval=None)
|
|
mem_mb = self.psutil_proc.memory_info().rss / (1024 * 1024)
|
|
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
|
pass
|
|
|
|
return TaskMetrics(
|
|
cpu_percent=round(cpu, 1),
|
|
memory_mb=round(mem_mb, 1),
|
|
uptime_sec=round(uptime, 1),
|
|
pid=self.proc.pid,
|
|
restart_count=self.restart_count,
|
|
status=TaskStatus.RUNNING,
|
|
last_exit_code=self.last_exit_code,
|
|
last_error=self.last_error_message
|
|
)
|
|
|
|
def read_last_error(self, max_lines: int = 15) -> str:
|
|
if not self.stderr_path.exists():
|
|
return ""
|
|
try:
|
|
with open(self.stderr_path, "r", encoding="utf-8", errors="ignore") as f:
|
|
lines = f.readlines()
|
|
return "".join(lines[-max_lines:]).strip()
|
|
except Exception as e:
|
|
return f"Не удалось прочесть stderr: {e}"
|
|
|
|
def read_logs(self, max_lines: int = 30) -> Dict[str, List[str]]:
|
|
result = {"stdout": [], "stderr": []}
|
|
try:
|
|
if self.stdout_path.exists():
|
|
with open(self.stdout_path, "r", encoding="utf-8", errors="ignore") as f:
|
|
result["stdout"] = [line.rstrip() for line in f.readlines()[-max_lines:]]
|
|
if self.stderr_path.exists():
|
|
with open(self.stderr_path, "r", encoding="utf-8", errors="ignore") as f:
|
|
result["stderr"] = [line.rstrip() for line in f.readlines()[-max_lines:]]
|
|
except Exception as e:
|
|
log_err(f"Ошибка чтения логов {self.cfg.name}: {e}")
|
|
return result
|
|
|
|
def clear_logs(self):
|
|
with self._lock:
|
|
self._close_log_handles()
|
|
try:
|
|
if self.stdout_path.exists():
|
|
self.stdout_path.unlink()
|
|
if self.stderr_path.exists():
|
|
self.stderr_path.unlink()
|
|
# Удаляем также бэкапы
|
|
for p in self.logs_dir.glob(f"{self.cfg.name}_*.log.*"):
|
|
p.unlink(missing_ok=True)
|
|
log_info(f"Логи задачи '{self.cfg.name}' очищены")
|
|
except Exception as e:
|
|
log_err(f"Не удалось удалить логи {self.cfg.name}: {e}")
|