add multimodal quote handling for xiaoniu bot

plugins/ai_auto_response/main.py

@@ -1,5 +1,8 @@
 from __future__ import annotations
 
+import base64
+import html
+import imghdr
 import re
 import time
 import xml.etree.ElementTree as ET
@@ -137,6 +140,7 @@ class AIAutoResponsePlugin(MessagePluginInterface):
         sender = message.get("sender", "")
         bot: WechatAPIClient = message.get("bot")
         content = self._normalize_content(message)
+        quote_context = self._parse_quote_context(message.get("full_wx_msg"), room_id)
         sender_name = self._get_sender_name(room_id, sender)
         group_name = self._get_group_name(room_id, message)
         group_memory_profile = self.group_memory_service.build_group_memory_profile(room_id, group_name)
@@ -153,6 +157,7 @@ class AIAutoResponsePlugin(MessagePluginInterface):
             sharpness_style=group_profile.get("sharpness_style", ""),
             is_at=message.get("is_at", False),
             content_preview=self._preview(content),
+            quote_type=quote_context.get("quote_type_label", ""),
             msg_type=str(message.get("type")),
         )
 
@@ -236,6 +241,7 @@ class AIAutoResponsePlugin(MessagePluginInterface):
         vector_memories = []
         if self.vector_memory.should_search(reply_mode, trigger.trigger_type, memory_hints.get("returning_member_state", "")):
             vector_memories = self.vector_memory.search(content, room_id, sender)
+        image_urls = await self._prepare_quote_image_inputs(bot, quote_context)
         self._log_event(
             "context",
             room_id=room_id,
@@ -246,6 +252,7 @@ class AIAutoResponsePlugin(MessagePluginInterface):
             reply_mode=reply_mode,
             recent_message_count=len(recent_messages),
             vector_hit_count=len(vector_memories),
+            image_input_count=len(image_urls),
         )
 
         context = self.context_builder.build(
@@ -260,11 +267,19 @@ class AIAutoResponsePlugin(MessagePluginInterface):
             flow_state=flow_state.state,
             reply_mode=reply_mode,
             vector_memories=vector_memories,
+            quote_context=quote_context | {"has_image_attachment": bool(image_urls)},
         )
 
         system_prompt = self.persona_engine.build_system_prompt(group_profile)
         user_prompt = self._build_user_prompt(context, memory_hints)
-        response = self._sanitize_response(self.llm_client.chat(system_prompt, user_prompt, user_id=f"{room_id}:{sender}"))
+        response = self._sanitize_response(
+            self.llm_client.chat(
+                system_prompt,
+                user_prompt,
+                user_id=f"{room_id}:{sender}",
+                image_urls=image_urls,
+            )
+        )
         if not response:
             self._log_event(
                 "model_empty",
@@ -361,6 +376,7 @@ class AIAutoResponsePlugin(MessagePluginInterface):
         return (
             f"当前群聊消息：\n{recent_text}\n\n"
             f"当前发言：{context.get('current_message', '')}\n"
+            f"引用补充：\n{context.get('quote_prompt', '') or '无'}\n"
             f"触发类型：{context.get('trigger_type', 'none')}\n"
             f"回复模式：{context.get('reply_mode', 'social_short')}\n"
             f"当前心流状态：{context.get('flow_state', 'idle')}\n"
@@ -561,6 +577,7 @@ class AIAutoResponsePlugin(MessagePluginInterface):
                 f"[XIAONIU] RECV room={room} user={sender_name}/{sender} "
                 f"at={self._yn(data.get('is_at'))} "
                 f"style={self._style_mark(data.get('humor_style', ''), data.get('sharpness_style', ''))} "
+                f"quote={data.get('quote_type', '-') or '-'} "
                 f"msg={data.get('content_preview', '')}"
             ).strip()
 
@@ -597,7 +614,8 @@ class AIAutoResponsePlugin(MessagePluginInterface):
                 f"mode={data.get('reply_mode', '')} "
                 f"acc={data.get('acceptance_state', '-') or '-'} "
                 f"recent={data.get('recent_message_count', 0)} "
-                f"vector={data.get('vector_hit_count', 0)}"
+                f"vector={data.get('vector_hit_count', 0)} "
+                f"img={data.get('image_input_count', 0)}"
             ).strip()
 
         if event == "model_empty":
@@ -644,3 +662,119 @@ class AIAutoResponsePlugin(MessagePluginInterface):
         humor = "humor" if "中等" in str(humor_style) or "偏上" in str(humor_style) else "plain"
         sharp = "sharp" if "毒舌" in str(sharpness_style) or "嘴欠" in str(sharpness_style) else "soft"
         return f"{humor}/{sharp}"
+
+    def _parse_quote_context(self, full_msg: Any, room_id: str) -> Dict[str, str]:
+        if not full_msg or not getattr(full_msg, "content", None):
+            return {}
+        xml_content = getattr(full_msg.content, "xml_content", "") or ""
+        if not xml_content:
+            return {}
+        try:
+            root = ET.fromstring(xml_content)
+        except ET.ParseError:
+            return {}
+
+        appmsg = root.find(".//appmsg")
+        if appmsg is None or appmsg.findtext("type", "").strip() != "57":
+            return {}
+
+        refer = appmsg.find("refermsg")
+        if refer is None:
+            return {}
+
+        title = html.unescape(appmsg.findtext("title", "") or "").strip()
+        quote_sender_name = html.unescape(refer.findtext("displayname", "") or "").strip()
+        if not quote_sender_name:
+            quote_sender = html.unescape(refer.findtext("chatusr", "") or "").strip()
+            quote_sender_name = self._get_sender_name(room_id, quote_sender) if quote_sender else "未知成员"
+        ref_type = int(refer.findtext("type", "0") or 0)
+        ref_content = html.unescape(refer.findtext("content", "") or "").strip()
+        quote_type_label = self._quote_type_label(ref_type)
+        quote_body = self._build_quote_body(ref_type, ref_content, title)
+        return {
+            "title": title,
+            "quote_sender_name": quote_sender_name,
+            "quote_type_label": quote_type_label,
+            "quote_body": quote_body,
+            "raw_ref_content": ref_content,
+        }
+
+    @staticmethod
+    def _quote_type_label(ref_type: int) -> str:
+        mapping = {
+            MessageType.TEXT.value: "引用文本",
+            MessageType.IMAGE.value: "引用图片",
+            MessageType.VIDEO.value: "引用视频",
+            MessageType.APP.value: "引用应用消息",
+            MessageType.EMOTICON.value: "引用表情",
+        }
+        return mapping.get(ref_type, f"引用消息[{ref_type}]")
+
+    @staticmethod
+    def _build_quote_body(ref_type: int, ref_content: str, title: str) -> str:
+        if ref_type == MessageType.TEXT.value:
+            return ref_content[:220].strip()
+        if ref_type == MessageType.IMAGE.value:
+            details = []
+            if title:
+                details.append(f"当前追问文案：{title}")
+            if ref_content:
+                details.append("被引用的是一张图片")
+            return "；".join(details) or "被引用的是一张图片"
+        if title:
+            return title[:220].strip()
+        return ref_content[:220].strip()
+
+    async def _prepare_quote_image_inputs(self, bot: WechatAPIClient, quote_context: Dict[str, str]) -> List[str]:
+        if not quote_context or quote_context.get("quote_type_label") != "引用图片":
+            return []
+        ref_content = quote_context.get("raw_ref_content", "") or ""
+        image_info = self._extract_quote_image_info(ref_content)
+        if not image_info:
+            return []
+        try:
+            base64_str = await bot.download_image(
+                aeskey=image_info["aeskey"],
+                cdnmidimgurl=image_info["url"],
+            )
+        except Exception as exc:
+            self._log_event("quote_image_fail", reason=f"download:{exc}")
+            return []
+        data_url = self._build_image_data_url(base64_str)
+        if not data_url:
+            self._log_event("quote_image_fail", reason="invalid_base64")
+            return []
+        return [data_url]
+
+    @staticmethod
+    def _extract_quote_image_info(ref_content: str) -> Dict[str, str]:
+        if not ref_content:
+            return {}
+        aeskey_match = re.search(r'aeskey="([^"]+)"', ref_content)
+        if not aeskey_match:
+            return {}
+        url_match = re.search(r'cdnmidimgurl="([^"]+)"', ref_content)
+        if not url_match:
+            url_match = re.search(r'cdnbigimgurl="([^"]+)"', ref_content)
+        if not url_match:
+            url_match = re.search(r'cdnthumburl="([^"]+)"', ref_content)
+        if not url_match:
+            return {}
+        return {
+            "aeskey": aeskey_match.group(1),
+            "url": url_match.group(1),
+        }
+
+    @staticmethod
+    def _build_image_data_url(base64_str: str) -> str:
+        raw_base64 = str(base64_str or "").strip()
+        if not raw_base64:
+            return ""
+        if "," in raw_base64 and raw_base64.startswith("data:"):
+            raw_base64 = raw_base64.split(",", 1)[1]
+        try:
+            image_bytes = base64.b64decode(raw_base64)
+        except Exception:
+            return ""
+        image_type = imghdr.what(None, h=image_bytes) or "jpeg"
+        return f"data:image/{image_type};base64,{raw_base64}"