新增定时任务重启漏执行补偿机制
变更项:1) 系统任务与插件调度重载后基于应执行时间和执行日志对账,判定是否漏执行。2) 仅在应执行时间已过且日志未覆盖时补跑一次,避免重复补偿。3) system_job_db 与 plugin_schedule_db 新增 get_latest_log_time 查询。4) 增加容差窗口与中文注释,降低误判概率。
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from loguru import logger
|
||||
@@ -18,6 +19,7 @@ class PluginScheduleManager:
|
||||
self.plugin_manager = plugin_manager
|
||||
self.db = plugin_schedule_db
|
||||
self._schedule_job_map: Dict[int, str] = {}
|
||||
self._compensation_tolerance_seconds = 120
|
||||
|
||||
def init_and_load(self):
|
||||
self.db.init_tables()
|
||||
@@ -134,6 +136,79 @@ class PluginScheduleManager:
|
||||
enabled_groups.append(gid)
|
||||
return enabled_groups
|
||||
|
||||
@staticmethod
|
||||
def _latest_expected_run_before_now(trigger_type: str, trigger_config: Dict[str, Any], now: datetime) -> datetime | None:
|
||||
cfg = trigger_config or {}
|
||||
if trigger_type == "every_seconds":
|
||||
seconds = int(cfg.get("seconds") or 0)
|
||||
if seconds <= 0:
|
||||
return None
|
||||
return now - timedelta(seconds=seconds)
|
||||
|
||||
if trigger_type == "at_times":
|
||||
time_list = cfg.get("time_list") or []
|
||||
candidates = []
|
||||
for text in time_list:
|
||||
try:
|
||||
tm = datetime.strptime(str(text), "%H:%M").time()
|
||||
except Exception:
|
||||
continue
|
||||
dt = datetime.combine(now.date(), tm)
|
||||
if dt > now:
|
||||
dt -= timedelta(days=1)
|
||||
candidates.append(dt)
|
||||
return max(candidates) if candidates else None
|
||||
|
||||
if trigger_type in ("every_weekday_time", "every_week_time"):
|
||||
try:
|
||||
weekday = int(cfg.get("weekday"))
|
||||
tm = datetime.strptime(str(cfg.get("time_str") or ""), "%H:%M").time()
|
||||
except Exception:
|
||||
return None
|
||||
days_ago = (now.weekday() - weekday + 7) % 7
|
||||
dt = datetime.combine((now - timedelta(days=days_ago)).date(), tm)
|
||||
if dt > now:
|
||||
dt -= timedelta(days=7)
|
||||
return dt
|
||||
|
||||
if trigger_type == "every_month_last_day_time":
|
||||
try:
|
||||
tm = datetime.strptime(str(cfg.get("time_str") or ""), "%H:%M").time()
|
||||
except Exception:
|
||||
return None
|
||||
if now.month == 12:
|
||||
next_month = datetime(now.year + 1, 1, 1)
|
||||
else:
|
||||
next_month = datetime(now.year, now.month + 1, 1)
|
||||
last_day = next_month - timedelta(days=1)
|
||||
dt = datetime.combine(last_day.date(), tm)
|
||||
if dt > now:
|
||||
if now.month == 1:
|
||||
prev_next_month = datetime(now.year, 1, 1)
|
||||
else:
|
||||
prev_next_month = datetime(now.year, now.month, 1)
|
||||
prev_last_day = prev_next_month - timedelta(days=1)
|
||||
dt = datetime.combine(prev_last_day.date(), tm)
|
||||
return dt
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _run_coro_blocking(coro):
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return asyncio.run(coro)
|
||||
return loop.create_task(coro)
|
||||
|
||||
def _should_compensate_once(self, schedule_id: int, trigger_type: str, trigger_config: Dict[str, Any]) -> bool:
|
||||
expected_at = self._latest_expected_run_before_now(trigger_type, trigger_config, datetime.now())
|
||||
if not expected_at:
|
||||
return False
|
||||
latest_log_at = self.db.get_latest_log_time(schedule_id)
|
||||
if not latest_log_at:
|
||||
return False
|
||||
return latest_log_at < (expected_at - timedelta(seconds=self._compensation_tolerance_seconds))
|
||||
|
||||
async def _run_one_schedule(self, schedule_row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
schedule_id = int(schedule_row["id"])
|
||||
action_key = schedule_row.get("action_key")
|
||||
@@ -202,6 +277,19 @@ class PluginScheduleManager:
|
||||
)
|
||||
self._schedule_job_map[schedule_id] = job_id
|
||||
|
||||
# 重启/重载补偿:如果最近一次应执行时间已过且日志未覆盖,补跑一次。
|
||||
try:
|
||||
trigger_type = row.get("trigger_type", "at_times")
|
||||
trigger_config = row.get("trigger_config", {"time_list": ["09:00"]})
|
||||
if self._should_compensate_once(schedule_id, trigger_type, trigger_config):
|
||||
logger.warning(
|
||||
f"插件调度触发漏执行补偿: schedule_id={schedule_id}, "
|
||||
f"plugin={row.get('plugin_name')}, action={row.get('action_key')}"
|
||||
)
|
||||
self._run_coro_blocking(_runner())
|
||||
except Exception as e:
|
||||
logger.error(f"插件调度漏执行补偿失败: schedule_id={schedule_id}, error={e}")
|
||||
|
||||
def list_schedules_with_runtime(self) -> List[Dict[str, Any]]:
|
||||
db_rows = self.db.list_schedules()
|
||||
runtime_rows = async_job.get_jobs_snapshot()
|
||||
|
||||
Reference in New Issue
Block a user