新增定时任务重启漏执行补偿机制

变更项:1) 系统任务与插件调度重载后基于应执行时间和执行日志对账,判定是否漏执行。2) 仅在应执行时间已过且日志未覆盖时补跑一次,避免重复补偿。3) system_job_db 与 plugin_schedule_db 新增 get_latest_log_time 查询。4) 增加容差窗口与中文注释,降低误判概率。
This commit is contained in:
liuwei
2026-04-17 09:38:15 +08:00
parent 6af91756d3
commit 3226fabcec
4 changed files with 216 additions and 2 deletions

View File

@@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from datetime import datetime
import asyncio
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional
from loguru import logger
@@ -18,6 +19,7 @@ class PluginScheduleManager:
self.plugin_manager = plugin_manager
self.db = plugin_schedule_db
self._schedule_job_map: Dict[int, str] = {}
self._compensation_tolerance_seconds = 120
def init_and_load(self):
self.db.init_tables()
@@ -134,6 +136,79 @@ class PluginScheduleManager:
enabled_groups.append(gid)
return enabled_groups
@staticmethod
def _latest_expected_run_before_now(trigger_type: str, trigger_config: Dict[str, Any], now: datetime) -> datetime | None:
cfg = trigger_config or {}
if trigger_type == "every_seconds":
seconds = int(cfg.get("seconds") or 0)
if seconds <= 0:
return None
return now - timedelta(seconds=seconds)
if trigger_type == "at_times":
time_list = cfg.get("time_list") or []
candidates = []
for text in time_list:
try:
tm = datetime.strptime(str(text), "%H:%M").time()
except Exception:
continue
dt = datetime.combine(now.date(), tm)
if dt > now:
dt -= timedelta(days=1)
candidates.append(dt)
return max(candidates) if candidates else None
if trigger_type in ("every_weekday_time", "every_week_time"):
try:
weekday = int(cfg.get("weekday"))
tm = datetime.strptime(str(cfg.get("time_str") or ""), "%H:%M").time()
except Exception:
return None
days_ago = (now.weekday() - weekday + 7) % 7
dt = datetime.combine((now - timedelta(days=days_ago)).date(), tm)
if dt > now:
dt -= timedelta(days=7)
return dt
if trigger_type == "every_month_last_day_time":
try:
tm = datetime.strptime(str(cfg.get("time_str") or ""), "%H:%M").time()
except Exception:
return None
if now.month == 12:
next_month = datetime(now.year + 1, 1, 1)
else:
next_month = datetime(now.year, now.month + 1, 1)
last_day = next_month - timedelta(days=1)
dt = datetime.combine(last_day.date(), tm)
if dt > now:
if now.month == 1:
prev_next_month = datetime(now.year, 1, 1)
else:
prev_next_month = datetime(now.year, now.month, 1)
prev_last_day = prev_next_month - timedelta(days=1)
dt = datetime.combine(prev_last_day.date(), tm)
return dt
return None
@staticmethod
def _run_coro_blocking(coro):
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return asyncio.run(coro)
return loop.create_task(coro)
def _should_compensate_once(self, schedule_id: int, trigger_type: str, trigger_config: Dict[str, Any]) -> bool:
expected_at = self._latest_expected_run_before_now(trigger_type, trigger_config, datetime.now())
if not expected_at:
return False
latest_log_at = self.db.get_latest_log_time(schedule_id)
if not latest_log_at:
return False
return latest_log_at < (expected_at - timedelta(seconds=self._compensation_tolerance_seconds))
async def _run_one_schedule(self, schedule_row: Dict[str, Any]) -> Dict[str, Any]:
schedule_id = int(schedule_row["id"])
action_key = schedule_row.get("action_key")
@@ -202,6 +277,19 @@ class PluginScheduleManager:
)
self._schedule_job_map[schedule_id] = job_id
# 重启/重载补偿:如果最近一次应执行时间已过且日志未覆盖,补跑一次。
try:
trigger_type = row.get("trigger_type", "at_times")
trigger_config = row.get("trigger_config", {"time_list": ["09:00"]})
if self._should_compensate_once(schedule_id, trigger_type, trigger_config):
logger.warning(
f"插件调度触发漏执行补偿: schedule_id={schedule_id}, "
f"plugin={row.get('plugin_name')}, action={row.get('action_key')}"
)
self._run_coro_blocking(_runner())
except Exception as e:
logger.error(f"插件调度漏执行补偿失败: schedule_id={schedule_id}, error={e}")
def list_schedules_with_runtime(self) -> List[Dict[str, Any]]:
db_rows = self.db.list_schedules()
runtime_rows = async_job.get_jobs_snapshot()