Dict[str, Any]:
- """构建插件执行保护的默认记录。"""
- return {
- "circuit_state": "closed",
- "consecutive_failures": 0,
- "consecutive_timeouts": 0,
- "failure_count_total": 0,
- "timeout_count_total": 0,
- "success_count_total": 0,
- "last_failure_at": 0.0,
- "last_success_at": 0.0,
- "last_error_message": "",
- "last_failure_type": "",
- "last_process_time_ms": 0.0,
- "last_timeout_seconds": 0,
- "circuit_opened_at": 0.0,
- "circuit_until": 0.0,
- "half_open_in_flight": False,
- }
-
- def _get_or_create_plugin_guard_record(self, module_name: str) -> Dict[str, Any]:
- """读取或初始化插件执行保护记录。"""
- if module_name not in self.plugin_guard_records:
- self.plugin_guard_records[module_name] = self._build_default_guard_record()
- return self.plugin_guard_records[module_name]
-
- def reset_plugin_guard_state(self, plugin_or_module_name) -> None:
- """重置插件执行保护状态。"""
- module_name = ""
- if isinstance(plugin_or_module_name, PluginInterface):
- module_name = self._get_module_name_from_plugin(plugin_or_module_name) or ""
- else:
- module_name = str(plugin_or_module_name or "").strip()
- if not module_name:
- return
-
- with self._guard_lock:
- self.plugin_guard_records[module_name] = self._build_default_guard_record()
-
- def get_plugin_guard_snapshot(self, plugin_or_module_name) -> Dict[str, Any]:
- """读取适合后台展示的插件执行保护快照。"""
- module_name = ""
- if isinstance(plugin_or_module_name, PluginInterface):
- module_name = self._get_module_name_from_plugin(plugin_or_module_name) or ""
- else:
- module_name = str(plugin_or_module_name or "").strip()
-
- if not module_name:
- return self._build_default_guard_record()
-
- with self._guard_lock:
- record = dict(self._get_or_create_plugin_guard_record(module_name))
-
- now_ts = time.time()
- circuit_until = float(record.get("circuit_until") or 0.0)
- open_remaining_seconds = max(0, int(circuit_until - now_ts)) if circuit_until > 0 else 0
- record["open_remaining_seconds"] = open_remaining_seconds
- return record
-
- def try_acquire_plugin_execution(
- self,
- plugin: PluginInterface,
- *,
- recovery_seconds: int,
- ) -> Dict[str, Any]:
- """判断插件当前是否允许继续执行。"""
- module_name = self._get_module_name_from_plugin(plugin) or plugin.name
- now_ts = time.time()
-
- with self._guard_lock:
- record = self._get_or_create_plugin_guard_record(module_name)
- circuit_state = str(record.get("circuit_state", "closed") or "closed").strip().lower()
- circuit_until = float(record.get("circuit_until") or 0.0)
- recovery_seconds = max(int(recovery_seconds or 0), 30)
-
- # 熔断打开期间直接拒绝新流量：
- # 1. 目标是尽快切断持续失败插件对主链路的影响；
- # 2. 若冷却时间未到，则所有新消息都应跳过该插件；
- # 3. 返回剩余秒数，便于日志和后台展示更可读。
- if circuit_state == "open" and circuit_until > now_ts:
- return {
- "allowed": False,
- "reason": "circuit_open",
- "open_remaining_seconds": max(0, int(circuit_until - now_ts)),
- "circuit_state": circuit_state,
- }
-
- # 熔断窗口已过，放一个半开探测请求：
- # 1. 只允许一个探测请求进入，避免刚恢复时被瞬时并发打爆；
- # 2. 成功则后续 record_success 会关闭熔断并清零计数；
- # 3. 失败则 record_failure 会重新打开熔断窗口。
- if circuit_state == "open" and circuit_until <= now_ts:
- record["circuit_state"] = "half_open"
- record["half_open_in_flight"] = True
- record["circuit_until"] = now_ts + recovery_seconds
- return {
- "allowed": True,
- "reason": "half_open_probe",
- "open_remaining_seconds": 0,
- "circuit_state": "half_open",
- }
-
- # 半开状态下，如果已经有探测请求在飞，就继续拒绝后续请求。
- if circuit_state == "half_open":
- if bool(record.get("half_open_in_flight", False)):
- return {
- "allowed": False,
- "reason": "half_open_probe_in_flight",
- "open_remaining_seconds": max(0, int(float(record.get("circuit_until") or 0.0) - now_ts)),
- "circuit_state": "half_open",
- }
- record["half_open_in_flight"] = True
- return {
- "allowed": True,
- "reason": "half_open_probe",
- "open_remaining_seconds": 0,
- "circuit_state": "half_open",
- }
-
- return {
- "allowed": True,
- "reason": "closed",
- "open_remaining_seconds": 0,
- "circuit_state": "closed",
- }
-
- def record_plugin_execution_success(
- self,
- plugin: PluginInterface,
- *,
- process_time_ms: float,
- ) -> None:
- """记录插件一次成功执行。"""
- module_name = self._get_module_name_from_plugin(plugin) or plugin.name
- now_ts = time.time()
-
- with self._guard_lock:
- record = self._get_or_create_plugin_guard_record(module_name)
- record["success_count_total"] = int(record.get("success_count_total", 0) or 0) + 1
- record["consecutive_failures"] = 0
- record["consecutive_timeouts"] = 0
- record["last_success_at"] = now_ts
- record["last_process_time_ms"] = float(process_time_ms or 0.0)
- record["last_failure_type"] = ""
- record["last_error_message"] = ""
- record["last_timeout_seconds"] = 0
- record["half_open_in_flight"] = False
- # 半开探测成功后立即闭合熔断：
- # 1. 说明插件已经至少恢复到可以完成一次请求；
- # 2. 这里不保留历史 consecutive_failures，避免恢复后仍长期背着旧债；
- # 3. 同时把 open 时间清空，后台能立即看到状态回归 closed。
- record["circuit_state"] = "closed"
- record["circuit_opened_at"] = 0.0
- record["circuit_until"] = 0.0
-
- def record_plugin_execution_failure(
- self,
- plugin: PluginInterface,
- *,
- failure_type: str,
- error_message: str,
- process_time_ms: float,
- timeout_seconds: int,
- failure_threshold: int,
- recovery_seconds: int,
- ) -> Dict[str, Any]:
- """记录插件一次失败执行，并根据连续失败次数决定是否打开熔断。"""
- module_name = self._get_module_name_from_plugin(plugin) or plugin.name
- now_ts = time.time()
- normalized_failure_type = str(failure_type or "error").strip().lower() or "error"
- failure_threshold = max(int(failure_threshold or 0), 2)
- recovery_seconds = max(int(recovery_seconds or 0), 30)
-
- with self._guard_lock:
- record = self._get_or_create_plugin_guard_record(module_name)
- record["failure_count_total"] = int(record.get("failure_count_total", 0) or 0) + 1
- record["consecutive_failures"] = int(record.get("consecutive_failures", 0) or 0) + 1
- record["last_failure_at"] = now_ts
- record["last_failure_type"] = normalized_failure_type
- record["last_error_message"] = str(error_message or "").strip()
- record["last_process_time_ms"] = float(process_time_ms or 0.0)
- record["last_timeout_seconds"] = int(timeout_seconds or 0)
- record["half_open_in_flight"] = False
-
- if normalized_failure_type == "timeout":
- record["timeout_count_total"] = int(record.get("timeout_count_total", 0) or 0) + 1
- record["consecutive_timeouts"] = int(record.get("consecutive_timeouts", 0) or 0) + 1
- else:
- record["consecutive_timeouts"] = 0
-
- should_open_circuit = (
- str(record.get("circuit_state", "closed") or "closed").strip().lower() == "half_open"
- or int(record.get("consecutive_failures", 0) or 0) >= failure_threshold
- )
-
- if should_open_circuit:
- record["circuit_state"] = "open"
- record["circuit_opened_at"] = now_ts
- record["circuit_until"] = now_ts + recovery_seconds
- else:
- record["circuit_state"] = "closed"
-
- return dict(record)
-
@staticmethod
def _is_sensitive_config_key(key: str) -> bool:
"""判断配置键是否属于敏感信息。"""
@@ -452,13 +242,11 @@ class PluginManager:
module_name: str,
config_overview: Dict[str, Any],
runtime_record: Dict[str, Any],
- guard_snapshot: Optional[Dict[str, Any]] = None,
) -> List[Dict[str, str]]:
"""根据插件元信息、配置和运行态生成治理诊断。"""
diagnostics = []
runtime_state = str(runtime_record.get("state", "") or "").strip().lower()
runtime_message = str(runtime_record.get("message", "") or "").strip()
- guard_snapshot = dict(guard_snapshot or {})
if runtime_state == "load_failed":
diagnostics.append(
@@ -504,34 +292,6 @@ class PluginManager:
)
)
- circuit_state = str(guard_snapshot.get("circuit_state", "closed") or "closed").strip().lower()
- consecutive_failures = int(guard_snapshot.get("consecutive_failures", 0) or 0)
- open_remaining_seconds = int(guard_snapshot.get("open_remaining_seconds", 0) or 0)
- if circuit_state == "open":
- diagnostics.append(
- self._build_diagnostic(
- "error",
- "circuit_open",
- f"插件保护熔断已打开，预计 {open_remaining_seconds} 秒后再尝试半开恢复。",
- )
- )
- elif circuit_state == "half_open":
- diagnostics.append(
- self._build_diagnostic(
- "warning",
- "circuit_half_open",
- "插件当前处于半开探测状态，正在等待恢复结果。",
- )
- )
- elif consecutive_failures > 0:
- diagnostics.append(
- self._build_diagnostic(
- "info",
- "recent_failures",
- f"插件最近存在连续失败记录，当前连续失败次数为 {consecutive_failures}。",
- )
- )
-
if plugin is None:
return diagnostics
@@ -606,7 +366,6 @@ class PluginManager:
"""为已加载插件生成标准治理快照。"""
module_name = self._get_module_name_from_plugin(plugin) or "unknown"
runtime_record = self._get_module_runtime_state(module_name)
- guard_snapshot = self.get_plugin_guard_snapshot(module_name)
config_path = plugin.get_config_path()
config_overview = self._read_plugin_config_overview(config_path)
commands = self._collect_plugin_commands(plugin)
@@ -617,7 +376,6 @@ class PluginManager:
module_name=module_name,
config_overview=config_overview,
runtime_record=runtime_record,
- guard_snapshot=guard_snapshot,
)
governance_summary = self._summarize_governance_status(governance_diagnostics)
@@ -647,13 +405,11 @@ class PluginManager:
"governance_info_count": governance_summary["info_count"],
"runtime_state": runtime_record.get("state", "loaded"),
"runtime_message": runtime_record.get("message", ""),
- "execution_guard": guard_snapshot,
}
def _build_unloaded_plugin_snapshot(self, module_name: str) -> Dict[str, Any]:
"""为未成功加载的插件模块生成治理快照。"""
runtime_record = self._get_module_runtime_state(module_name)
- guard_snapshot = self.get_plugin_guard_snapshot(module_name)
config_path = os.path.join(self.plugin_dir, module_name, "config.toml")
if not os.path.exists(config_path):
config_path = os.path.join(self.plugin_dir, f"{module_name}", "config.toml")
@@ -663,7 +419,6 @@ class PluginManager:
module_name=module_name,
config_overview=config_overview,
runtime_record=runtime_record,
- guard_snapshot=guard_snapshot,
)
governance_summary = self._summarize_governance_status(governance_diagnostics)
runtime_state = str(runtime_record.get("state", "") or "").strip().lower()
@@ -699,7 +454,6 @@ class PluginManager:
"governance_info_count": governance_summary["info_count"],
"runtime_state": runtime_state or "discovered",
"runtime_message": runtime_record.get("message", ""),
- "execution_guard": guard_snapshot,
}
@staticmethod
@@ -1039,7 +793,6 @@ class PluginManager:
self.module_to_display[module_name] = display_name
self.LOG.debug(f"PluginManager：添加缺失的模块映射 {module_name} -> {display_name}")
self._record_module_runtime_state(module_name, "loaded", "插件已在内存中复用现有实例。")
- self.reset_plugin_guard_state(module_name)
self._inject_bot_to_plugin(plugin)
return plugin
except Exception as e:
@@ -1119,7 +872,6 @@ class PluginManager:
self.module_to_display[module_name] = display_name
self._refresh_module_file_state(module_name)
self._record_module_runtime_state(module_name, "loaded", "插件已成功加载。")
- self.reset_plugin_guard_state(module_name)
# self.LOG.info(f"PluginManager：添加模块映射 {module_name} -> {display_name}")
return plugin
@@ -1175,7 +927,6 @@ class PluginManager:
self.module_to_display[module_name] = display_name
self._refresh_module_file_state(module_name)
self._record_module_runtime_state(module_name, "loaded", "插件已成功加载。")
- self.reset_plugin_guard_state(module_name)
# self.LOG.info(f"PluginManager：添加模块映射 {module_name} -> {display_name}")
return plugin
@@ -1310,7 +1061,6 @@ class PluginManager:
if plugin.start():
plugin.status = PluginStatus.RUNNING
module_name = self._get_module_name_from_plugin(plugin) or name
- self.reset_plugin_guard_state(module_name)
self._record_module_runtime_state(module_name, "running", "插件已启动并进入运行态。")
self.LOG.debug(f"PluginManager：插件 {display_name} 状态变更为在运行")
return True
diff --git a/docs/工程优化与Feature清单.md b/docs/工程优化与Feature清单.md
index 9e361c9..5b922d1 100644
--- a/docs/工程优化与Feature清单.md
+++ b/docs/工程优化与Feature清单.md
@@ -433,13 +433,6 @@
- 防止单插件问题拖垮整体系统
-当前进展：
-
-- 第一阶段已完成：消息插件执行已增加统一超时保护，避免单插件长时间卡住主链路
-- 第一阶段已完成：已补充连续失败熔断、冷却后半开探测与自动恢复逻辑
-- 第一阶段已完成：插件治理快照与后台详情已可查看执行保护状态、连续失败与恢复剩余时间
-- 后续可继续补充插件级并发配额、失败原因聚合、后台手动解除熔断与更细粒度的隔离策略
-
建议内容：
- 插件处理超时控制
diff --git a/robot.py b/robot.py
index 87e5d3a..912a3c7 100644
--- a/robot.py
+++ b/robot.py
@@ -649,33 +649,7 @@ class Robot:
# 检查插件是否可以处理该消息
if plugin.can_process(plugin_msg):
- protection_policy = self._build_message_plugin_protection_policy(plugin)
- acquire_result = self.plugin_manager.try_acquire_plugin_execution(
- plugin,
- recovery_seconds=protection_policy["circuit_recovery_seconds"],
- )
- if not acquire_result.get("allowed", False):
- # 熔断打开或半开探测占用时，这里只跳过当前插件：
- # 1. 保护目标是避免单插件持续拖慢主链路，而不是直接关闭整个插件；
- # 2. 后续插件仍然可以继续尝试处理当前消息，降低功能面损失；
- # 3. 冷却结束后会自动进入半开恢复探测，无需人工介入恢复。
- self.LOG.warning(
- self._trace_message(
- msg,
- f"插件保护跳过 plugin={plugin.name} reason={acquire_result.get('reason')} "
- f"remaining={acquire_result.get('open_remaining_seconds', 0)}s"
- )
- )
- continue
-
- processed, _ = await asyncio.wait_for(
- plugin.process_message(plugin_msg),
- timeout=protection_policy["process_timeout_seconds"],
- )
- self.plugin_manager.record_plugin_execution_success(
- plugin,
- process_time_ms=self._elapsed_ms(started_at),
- )
+ processed, _ = await plugin.process_message(plugin_msg)
self._record_plugin_call_result(
plugin=plugin,
msg=msg,
@@ -696,58 +670,14 @@ class Robot:
)
)
return True
- except asyncio.TimeoutError as timeout_error:
- protection_policy = self._build_message_plugin_protection_policy(plugin)
- failure_record = self.plugin_manager.record_plugin_execution_failure(
- plugin,
- failure_type="timeout",
- error_message=(
- f"插件执行超时，超过 {protection_policy['process_timeout_seconds']} 秒仍未完成。"
- ),
- process_time_ms=self._elapsed_ms(started_at),
- timeout_seconds=protection_policy["process_timeout_seconds"],
- failure_threshold=protection_policy["failure_threshold"],
- recovery_seconds=protection_policy["circuit_recovery_seconds"],
- )
- self._record_plugin_call_error(
- plugin=plugin,
- msg=msg,
- command_name=command_name,
- error=timeout_error,
- )
- self.LOG.error(
- self._trace_message(
- msg,
- f"插件 {plugin.name} 执行超时，timeout={protection_policy['process_timeout_seconds']}s "
- f"circuit_state={failure_record.get('circuit_state')} "
- f"consecutive_failures={failure_record.get('consecutive_failures')}"
- )
- )
except Exception as e:
- protection_policy = self._build_message_plugin_protection_policy(plugin)
- failure_record = self.plugin_manager.record_plugin_execution_failure(
- plugin,
- failure_type="error",
- error_message=str(e),
- process_time_ms=self._elapsed_ms(started_at),
- timeout_seconds=0,
- failure_threshold=protection_policy["failure_threshold"],
- recovery_seconds=protection_policy["circuit_recovery_seconds"],
- )
self._record_plugin_call_error(
plugin=plugin,
msg=msg,
command_name=command_name,
error=e,
)
- self.LOG.error(
- self._trace_message(
- msg,
- f"插件 {plugin.name} 处理消息失败: {e} "
- f"circuit_state={failure_record.get('circuit_state')} "
- f"consecutive_failures={failure_record.get('consecutive_failures')}"
- )
- )
+ self.LOG.error(self._trace_message(msg, f"插件 {plugin.name} 处理消息失败: {e}"))
return False
@@ -796,70 +726,6 @@ class Robot:
msg_type = getattr(getattr(msg, "msg_type", None), "name", "")
return f"[{msg_type or 'UNKNOWN'}]"
- @staticmethod
- def _safe_positive_int(value, default: int) -> int:
- """把配置中的数字安全转成正整数。"""
- try:
- parsed = int(value)
- return parsed if parsed > 0 else default
- except (TypeError, ValueError):
- return default
-
- def _build_message_plugin_protection_policy(self, plugin) -> dict:
- """构建消息插件执行保护策略。"""
- plugin_config = getattr(plugin, "_config", {}) or {}
- runtime_config = plugin_config.get("runtime", {}) if isinstance(plugin_config, dict) else {}
- runtime_config = runtime_config if isinstance(runtime_config, dict) else {}
- breaker_config = runtime_config.get("circuit_breaker", {}) if isinstance(runtime_config, dict) else {}
- breaker_config = breaker_config if isinstance(breaker_config, dict) else {}
-
- # 超时策略尽量遵循“显式配置优先，已有内部超时参数兜底”的思路：
- # 1. 新插件如果有特殊需求，只需要在 runtime / circuit_breaker 下声明自己的超时；
- # 2. 老插件不改代码也能自动复用现有的 request / llm / render 超时字段；
- # 3. 最终统一加一个缓冲区，避免外层 wait_for 比插件内部自己的超时还更早打断。
- explicit_timeout = (
- runtime_config.get("plugin_process_timeout_seconds")
- or runtime_config.get("message_timeout_seconds")
- or breaker_config.get("timeout_seconds")
- or getattr(plugin, "plugin_process_timeout_seconds", 0)
- )
- timeout_candidates = []
- for attr_name in [
- "llm_call_timeout_sec",
- "_request_timeout_seconds",
- "default_timeout",
- "_image_render_timeout_seconds",
- "image_render_timeout_seconds",
- "_receive_timeout",
- "_connect_timeout_seconds",
- "_connect_timeout",
- ]:
- attr_value = getattr(plugin, attr_name, 0)
- if isinstance(attr_value, (int, float)) and attr_value > 0:
- timeout_candidates.append(int(attr_value))
-
- if explicit_timeout:
- resolved_timeout = self._safe_positive_int(explicit_timeout, 30)
- elif timeout_candidates:
- resolved_timeout = max(timeout_candidates) + 10
- else:
- resolved_timeout = 30
-
- failure_threshold = self._safe_positive_int(
- breaker_config.get("failure_threshold") or runtime_config.get("circuit_breaker_failure_threshold") or 3,
- 3,
- )
- circuit_recovery_seconds = self._safe_positive_int(
- breaker_config.get("recovery_seconds") or runtime_config.get("circuit_breaker_recovery_seconds") or 180,
- 180,
- )
-
- return {
- "process_timeout_seconds": max(10, min(int(resolved_timeout), 180)),
- "failure_threshold": max(2, min(int(failure_threshold), 10)),
- "circuit_recovery_seconds": max(30, min(int(circuit_recovery_seconds), 900)),
- }
-
def _get_stats_collector_plugin(self):
"""获取运行中的统计收集插件实例。"""
# 统计插件已经从“事件订阅”切到“主链路直接回调”，