优化斗鱼弹幕日报LLM入参压缩
- 新增面向LLM的用户索引、时间线事件块与整句高频线索压缩结构 - 将弹幕时间统一压缩为日期加时分并抽离UID尾号、牌子等级等重复元信息 - 下调中文分词结果在提示材料中的权重，改为优先使用现场原句和时间线材料
This commit is contained in:
liuwei
+89
-5
@@ -2043,6 +2043,7 @@ class DouyuPlugin(MessagePluginInterface):
|
||||
return None
|
||||
|
||||
session_payloads: List[Dict[str, Any]] = []
|
||||
all_messages: List[Dict[str, Any]] = []
|
||||
total_message_count = 0
|
||||
total_noise_filtered_count = 0
|
||||
total_organized_message_count = 0
|
||||
@@ -2077,6 +2078,9 @@ class DouyuPlugin(MessagePluginInterface):
|
||||
messages = DouyuDanmuSummaryHelper.load_session_messages(room_id, session)
|
||||
if len(messages) < self._daily_report_min_messages:
|
||||
continue
|
||||
# 这里额外保留一份“日报级全量消息集合”，
|
||||
# 后面统一做面向 LLM 的压缩，避免按 session 先压一次、汇总时再拼回去导致结构发散。
|
||||
all_messages.extend(messages)
|
||||
payload = DouyuDanmuSummaryHelper.build_llm_payload(room_id, session, messages)
|
||||
session_payloads.append(payload)
|
||||
meta = payload.get("session_meta", {}) or {}
|
||||
@@ -2169,6 +2173,14 @@ class DouyuPlugin(MessagePluginInterface):
|
||||
os.makedirs(artifact_dir, exist_ok=True)
|
||||
audience_trend = self._build_audience_trend(sessions)
|
||||
room_context = self._build_room_semantic_context(room_id, nickname, room_name, sessions)
|
||||
llm_compact = DouyuDanmuSummaryHelper.build_compact_prompt_assets(
|
||||
all_messages,
|
||||
bucket_minutes=5,
|
||||
speaker_limit=80,
|
||||
timeline_limit=24,
|
||||
samples_per_bucket=6,
|
||||
cue_limit=18,
|
||||
)
|
||||
payload = {
|
||||
"report_meta": {
|
||||
"room_id": room_id,
|
||||
@@ -2227,6 +2239,11 @@ class DouyuPlugin(MessagePluginInterface):
|
||||
"chronological_samples": chronological_samples[:40],
|
||||
# 多场直播时保留每一场的轻量故事线，让粉丝日报更容易写出真正的“回放感”。
|
||||
"session_storylines": session_storylines[:6],
|
||||
# 专供 LLM 的压缩材料：
|
||||
# 1. speaker_index 把 UID/牌子/等级从逐条消息里抽离；
|
||||
# 2. timeline_digest 用“时间块 + 原句样本 + 复读线索”还原现场；
|
||||
# 3. content_cues 不走中文分词，尽量保留整句高频信息。
|
||||
"llm_compact": llm_compact,
|
||||
}
|
||||
artifact_path = os.path.join(artifact_dir, f"{room_id}_{anchor_day.replace('-', '')}_daily_report_payload.json")
|
||||
with open(artifact_path, "w", encoding="utf-8") as f:
|
||||
@@ -2251,7 +2268,8 @@ class DouyuPlugin(MessagePluginInterface):
|
||||
"5. 单独列出 2-3 个热点时段。\n"
|
||||
"6. 整体控制在 600 字以内。\n\n"
|
||||
f"{room_context_prompt}"
|
||||
"下面是已经提纯给 LLM 的材料，优先依据现场弹幕片段、热点窗口和共识梗来写，不要被大段统计信息带偏。\n"
|
||||
"下面是已经提纯给 LLM 的材料，其中 `compact_scene_material` 是主阅读区：\n"
|
||||
"请优先依据其中的用户索引、时间线块、整句复读线索和原声样本来写，不要被大段统计信息带偏。\n"
|
||||
f"材料如下：\n{json.dumps(prompt_material, ensure_ascii=False, indent=2)}"
|
||||
)
|
||||
return system_prompt, user_prompt
|
||||
@@ -2276,7 +2294,8 @@ class DouyuPlugin(MessagePluginInterface):
|
||||
f"主播：{meta.get('nickname') or meta.get('room_name') or meta.get('room_id')}\n"
|
||||
f"日期：{meta.get('anchor_day', '')}\n"
|
||||
f"{room_context_prompt}"
|
||||
"下面是已经提纯给 LLM 的现场材料，请优先使用原声弹幕、热点窗口和复读梗，不要写成词频复述。\n"
|
||||
"下面是已经提纯给 LLM 的现场材料，请优先阅读 `compact_scene_material` 中的时间线块、整句复读线索和原声样本，"
|
||||
"不要写成词频复述。\n"
|
||||
f"材料：\n{json.dumps(prompt_material, ensure_ascii=False, indent=2)}"
|
||||
)
|
||||
return system_prompt, user_prompt
|
||||
@@ -2308,7 +2327,8 @@ class DouyuPlugin(MessagePluginInterface):
|
||||
f"主播：{meta.get('nickname') or meta.get('room_name') or meta.get('room_id')}\n"
|
||||
f"日期：{meta.get('anchor_day', '')}\n"
|
||||
f"{room_context_prompt}"
|
||||
"下面是已经提纯给 LLM 的现场材料，请优先抓原声弹幕、热点窗口和集体起哄片段，少写空泛概括。\n"
|
||||
"下面是已经提纯给 LLM 的现场材料，请优先抓 `compact_scene_material` 里的原声弹幕、时间线块和集体起哄片段，"
|
||||
"少写空泛概括。\n"
|
||||
f"材料：\n{json.dumps(prompt_material, ensure_ascii=False, indent=2)}"
|
||||
)
|
||||
return system_prompt, user_prompt
|
||||
@@ -2338,6 +2358,10 @@ class DouyuPlugin(MessagePluginInterface):
|
||||
peak_buckets = payload.get("peak_buckets", []) or []
|
||||
chronological_samples = payload.get("chronological_samples", []) or []
|
||||
session_storylines = payload.get("session_storylines", []) or []
|
||||
llm_compact = payload.get("llm_compact", {}) or {}
|
||||
speaker_index = llm_compact.get("speaker_index", []) or []
|
||||
timeline_digest = llm_compact.get("timeline_digest", []) or []
|
||||
content_cues = llm_compact.get("content_cues", []) or []
|
||||
|
||||
material: Dict[str, Any] = {
|
||||
"report_meta": {
|
||||
@@ -2358,6 +2382,64 @@ class DouyuPlugin(MessagePluginInterface):
|
||||
"storyline_keywords": self._normalize_text_list(room_context.get("storyline_keywords"))[:10],
|
||||
"style_hints": self._normalize_text_list(room_context.get("style_hints"))[:6],
|
||||
},
|
||||
# 这是新的主材料层，优先级高于传统的 top_terms：
|
||||
# 1. speaker_index 负责承接用户画像，避免在每条样本里重复塞 UUID/牌子/等级；
|
||||
# 2. timeline_digest 让模型按时间推进理解“哪一段开始起哄、哪一段反复刷屏”；
|
||||
# 3. content_cues 保留整句/短句级复读内容，不再依赖中文切词。
|
||||
"compact_scene_material": {
|
||||
"speaker_index": [
|
||||
{
|
||||
"speaker_id": str(item.get("speaker_id") or "").strip(),
|
||||
"nickname": str(item.get("nickname") or "").strip(),
|
||||
"uid_tail": str(item.get("uid_tail") or "").strip(),
|
||||
"badge_name": str(item.get("badge_name") or "").strip(),
|
||||
"badge_level": int(item.get("badge_level", 0) or 0),
|
||||
"room_level": int(item.get("room_level", 0) or 0),
|
||||
"noble_name": str(item.get("noble_name") or "").strip(),
|
||||
"message_count": int(item.get("message_count", 0) or 0),
|
||||
}
|
||||
for item in speaker_index[:40]
|
||||
if str(item.get("speaker_id") or "").strip()
|
||||
],
|
||||
"content_cues": [
|
||||
{
|
||||
"kind": str(item.get("kind") or "").strip(),
|
||||
"text": str(item.get("text") or "").strip()[:90],
|
||||
"count": int(item.get("count", 0) or 0),
|
||||
"user_count": int(item.get("user_count", 0) or 0),
|
||||
}
|
||||
for item in content_cues[:18]
|
||||
if str(item.get("text") or "").strip()
|
||||
],
|
||||
"timeline_digest": [
|
||||
{
|
||||
"date": str(item.get("date") or "").strip(),
|
||||
"start_hm": str(item.get("start_hm") or "").strip(),
|
||||
"message_count": int(item.get("message_count", 0) or 0),
|
||||
"user_count": int(item.get("user_count", 0) or 0),
|
||||
"repeated_cues": [
|
||||
{
|
||||
"text": str(cue.get("text") or "").strip()[:80],
|
||||
"count": int(cue.get("count", 0) or 0),
|
||||
"user_count": int(cue.get("user_count", 0) or 0),
|
||||
}
|
||||
for cue in (item.get("repeated_cues", []) or [])[:3]
|
||||
if str(cue.get("text") or "").strip()
|
||||
],
|
||||
"samples": [
|
||||
{
|
||||
"speaker_id": str(sample.get("speaker_id") or "").strip(),
|
||||
"hm": str(sample.get("hm") or "").strip(),
|
||||
"content": str(sample.get("content") or "").strip()[:90],
|
||||
}
|
||||
for sample in (item.get("samples", []) or [])[:6]
|
||||
if str(sample.get("content") or "").strip()
|
||||
],
|
||||
}
|
||||
for item in timeline_digest[:20]
|
||||
if (item.get("samples") or item.get("repeated_cues"))
|
||||
],
|
||||
},
|
||||
"session_overview": [
|
||||
{
|
||||
"session_id": str(item.get("session_id") or "").strip(),
|
||||
@@ -2368,9 +2450,11 @@ class DouyuPlugin(MessagePluginInterface):
|
||||
for item in sessions[:4]
|
||||
],
|
||||
"high_frequency_topics": {
|
||||
"top_terms": [
|
||||
# 这里刻意不再把中文分词结果作为主字段喂给 LLM，
|
||||
# 避免模型把碎词误当成主线；真正的内容理解优先走 compact_scene_material。
|
||||
"top_terms_legacy": [
|
||||
{"term": str(item.get("term") or "").strip(), "count": int(item.get("count", 0) or 0)}
|
||||
for item in top_terms[:16]
|
||||
for item in top_terms[:8]
|
||||
if str(item.get("term") or "").strip()
|
||||
],
|
||||
"burst_terms": [
|
||||
|
||||
Reference in New Issue
Block a user