liuwei
@@ -164,14 +164,13 @@ class DouyuDanmuSummaryHelper:
|
||||
面向 LLM 的高保真弹幕载荷。
|
||||
规则：
|
||||
1. 仅过滤平台/机器人类系统噪音。
|
||||
2. 只对“同内容重复弹幕”做合并，不再做偏激进的模板化折叠。
|
||||
3. 其他不同内容尽量保留，让模型看到更接近当晚现场的讨论全貌。
|
||||
2. 相同或高度模板化的内容做聚合，不直接删除。
|
||||
3. 其他不同内容尽量保留，并按时段/热点组织给模型。
|
||||
"""
|
||||
normalized = [item for item in messages if item and item.get("content")]
|
||||
prepared = cls._prepare_messages_for_llm(normalized)
|
||||
source_messages = prepared["llm_source_messages"]
|
||||
prepared = cls._prepare_messages(normalized)
|
||||
organized_messages = prepared["organized_messages"]
|
||||
bucket_stats = cls._build_time_buckets(source_messages, minutes=bucket_minutes)
|
||||
bucket_stats = cls._build_time_buckets(organized_messages, minutes=bucket_minutes)
|
||||
peak_buckets = sorted(
|
||||
bucket_stats,
|
||||
key=lambda item: item.get("message_count", 0),
|
||||
@@ -200,30 +199,31 @@ class DouyuDanmuSummaryHelper:
|
||||
"operator_metrics": cls._build_operator_metrics(normalized, organized_messages),
|
||||
"cleaning_rules": [
|
||||
"仅过滤系统噪音、机器人探测、平台提示类弹幕。",
|
||||
"只合并同内容重复弹幕，保留出现次数、人数、首末时间。",
|
||||
"不同句式、不同观点、不同刀圈讨论尽量原样保留，不再做模板化压缩。",
|
||||
"热点时段、顺时序样本和原声片段共同保留，方便 LLM 还原完整语境。",
|
||||
"明显重复的长模板文案按内容聚合，保留出现次数、人数、首末时间。",
|
||||
"其他相同内容按重复短语归并，但不抹掉不同观点和不同句式。",
|
||||
"高峰时段补充原始弹幕样本，方便 LLM 还原语境。",
|
||||
],
|
||||
# 字段名继续沿用 merged_templates / repeated_messages，
|
||||
# 目的是兼容下游模板和主流程，实际语义已经切换成“同内容重复弹幕聚合结果”。
|
||||
"merged_templates": prepared["duplicate_groups"][:top_repeat_count],
|
||||
"repeated_messages": prepared["duplicate_groups"][:top_repeat_count],
|
||||
"top_terms": cls._extract_top_terms(source_messages, limit=40),
|
||||
"burst_terms": cls._build_burst_terms(source_messages),
|
||||
"merged_templates": prepared["merged_templates"],
|
||||
"repeated_messages": cls._build_repeated_messages(
|
||||
organized_messages,
|
||||
limit=top_repeat_count,
|
||||
),
|
||||
"top_terms": cls._extract_top_terms(organized_messages, limit=30),
|
||||
"burst_terms": cls._build_burst_terms(organized_messages),
|
||||
"peak_buckets": cls._simplify_peak_buckets(peak_buckets),
|
||||
"representative_messages": cls._pick_representative_messages(organized_messages, bucket_stats),
|
||||
"raw_window_samples": cls._build_raw_window_samples(peak_buckets, per_bucket_limit=12),
|
||||
"raw_window_samples": cls._build_raw_window_samples(peak_buckets, per_bucket_limit=8),
|
||||
# 给日报类 LLM 再补一层“按时间推进的现场切片”。
|
||||
# 这样模型除了看热点窗口，还能顺着时间线理解气氛如何起、如何变、最后怎么收，
|
||||
# 对粉丝日报这类强调“节目效果”和“接梗链路”的文本尤其有帮助。
|
||||
"chronological_samples": cls._build_chronological_samples(organized_messages, limit=28),
|
||||
"chronological_samples": cls._build_chronological_samples(organized_messages, limit=20),
|
||||
# 每个 session 单独给一个轻量摘要，避免多场直播合并后，
|
||||
# 模型只看到全局热点而丢失“第一场在聊什么、第二场为什么突然转节奏”的信息。
|
||||
"session_storyline": cls._build_session_storyline(
|
||||
source_messages,
|
||||
organized_messages,
|
||||
bucket_stats,
|
||||
top_terms_limit=12,
|
||||
sample_limit=14,
|
||||
top_terms_limit=8,
|
||||
sample_limit=10,
|
||||
),
|
||||
}
|
||||
|
||||
@@ -586,36 +586,6 @@ class DouyuDanmuSummaryHelper:
|
||||
"organized_messages": organized_messages,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _prepare_messages_for_llm(cls, messages: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""
|
||||
为日报/粉丝日报的 LLM 链路做更保守的清洗。
|
||||
设计原则：
|
||||
1. 只过滤系统噪音，不再做“像模板就折叠掉”的强压缩；
|
||||
2. 只对完全同内容的重复弹幕做合并，避免复读海啸把语料挤爆；
|
||||
3. 其他不同表达、不同观点、不同讨论方向尽量完整保留。
|
||||
"""
|
||||
noise_messages: List[Dict[str, Any]] = []
|
||||
candidate_messages: List[Dict[str, Any]] = []
|
||||
for item in messages:
|
||||
content = str(item.get("content") or "").strip()
|
||||
if not content:
|
||||
continue
|
||||
if cls._is_noise_message(content):
|
||||
noise_messages.append(item)
|
||||
continue
|
||||
candidate_messages.append(item)
|
||||
|
||||
duplicate_groups, organized_messages = cls._merge_exact_duplicate_messages(candidate_messages)
|
||||
return {
|
||||
"noise_messages": noise_messages,
|
||||
# llm_source_messages 保留所有非噪音原始消息，用于词频、热点时段和讨论热度统计。
|
||||
"llm_source_messages": candidate_messages,
|
||||
# organized_messages 只去掉完全重复内容，用于给模型喂更丰富但不至于刷屏的原声样本。
|
||||
"organized_messages": organized_messages,
|
||||
"duplicate_groups": duplicate_groups,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _is_noise_message(cls, content: str) -> bool:
|
||||
text = str(content or "").strip()
|
||||
@@ -666,52 +636,6 @@ class DouyuDanmuSummaryHelper:
|
||||
organized_messages.sort(key=lambda item: item.get("timestamp") or datetime.min)
|
||||
return merged_templates[:20], organized_messages
|
||||
|
||||
@classmethod
|
||||
def _merge_exact_duplicate_messages(cls, messages: List[Dict[str, Any]]) -> (List[Dict[str, Any]], List[Dict[str, Any]]):
|
||||
"""
|
||||
只合并“同内容重复弹幕”。
|
||||
注意这里的归一化非常保守：
|
||||
1. 只处理大小写、空白和零宽字符；
|
||||
2. 不再去掉标点，不再做模板化抽象；
|
||||
3. 目的是尽量保留不同句式、不同观点和不同刀圈细节。
|
||||
"""
|
||||
grouped_messages: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||||
ordered_keys: List[str] = []
|
||||
for item in messages:
|
||||
normalized = cls._normalize_duplicate_text(str(item.get("content") or ""))
|
||||
if not normalized:
|
||||
continue
|
||||
if normalized not in grouped_messages:
|
||||
ordered_keys.append(normalized)
|
||||
grouped_messages[normalized].append(item)
|
||||
|
||||
duplicate_groups: List[Dict[str, Any]] = []
|
||||
organized_messages: List[Dict[str, Any]] = []
|
||||
for normalized in ordered_keys:
|
||||
items = grouped_messages[normalized]
|
||||
first = items[0]
|
||||
copied = dict(first)
|
||||
copied["repeat_count"] = len(items)
|
||||
copied["repeat_user_count"] = len({
|
||||
str(item.get("uid") or "") for item in items if str(item.get("uid") or "").strip()
|
||||
})
|
||||
organized_messages.append(copied)
|
||||
if len(items) < 2:
|
||||
continue
|
||||
duplicate_groups.append({
|
||||
"text": str(first.get("content") or "").strip()[:120],
|
||||
"count": len(items),
|
||||
"user_count": len({
|
||||
str(item.get("uid") or "") for item in items if str(item.get("uid") or "").strip()
|
||||
}),
|
||||
"first_time": str(first.get("timestamp_text") or ""),
|
||||
"last_time": str(items[-1].get("timestamp_text") or ""),
|
||||
})
|
||||
|
||||
duplicate_groups.sort(key=lambda item: item.get("count", 0), reverse=True)
|
||||
organized_messages.sort(key=lambda item: item.get("timestamp") or datetime.min)
|
||||
return duplicate_groups[:80], organized_messages
|
||||
|
||||
@classmethod
|
||||
def _build_repeated_messages(cls, messages: List[Dict[str, Any]], limit: int = 24) -> List[Dict[str, Any]]:
|
||||
grouped: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||||
@@ -743,22 +667,6 @@ class DouyuDanmuSummaryHelper:
|
||||
repeated_messages.sort(key=lambda item: item.get("count", 0), reverse=True)
|
||||
return repeated_messages[:limit]
|
||||
|
||||
@staticmethod
|
||||
def _normalize_duplicate_text(content: str) -> str:
|
||||
"""
|
||||
LLM 清洗链路的“同内容”判定保持保守。
|
||||
只做最小化标准化，避免把本来不同的表达误并成一类：
|
||||
1. 转小写；
|
||||
2. 去零宽字符；
|
||||
3. 折叠空白。
|
||||
"""
|
||||
text = str(content or "").strip().lower()
|
||||
if not text:
|
||||
return ""
|
||||
text = text.replace("\u200b", "").replace("\ufeff", "")
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def _build_burst_terms(cls, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
counters: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
Reference in New Issue
Block a user