687 lines
30 KiB
Python
687 lines
30 KiB
Python
from collections import OrderedDict
|
||
from typing import Any, Callable, Dict, List, Optional, Set, TypedDict
|
||
|
||
import asyncio
|
||
import json
|
||
import random
|
||
|
||
from json_repair import repair_json
|
||
from sqlmodel import select
|
||
|
||
from src.common.data_models.jargon_data_model import MaiJargon
|
||
from src.common.data_models.llm_service_data_models import LLMGenerationOptions
|
||
from src.common.database.database import get_db_session
|
||
from src.common.database.database_model import Jargon
|
||
from src.common.logger import get_logger
|
||
from src.config.config import global_config
|
||
from src.plugin_runtime.hook_schema_utils import build_object_schema
|
||
from src.plugin_runtime.host.hook_spec_registry import HookSpec, HookSpecRegistry
|
||
from src.prompt.prompt_manager import prompt_manager
|
||
from src.services.llm_service import LLMServiceClient
|
||
|
||
from .expression_utils import is_single_char_jargon
|
||
|
||
logger = get_logger("jargon")
|
||
|
||
llm_inference = LLMServiceClient(task_name="utils", request_type="jargon.inference")
|
||
|
||
|
||
class JargonEntry(TypedDict):
|
||
content: str
|
||
raw_content: Set[str]
|
||
|
||
|
||
class JargonMeaningEntry(TypedDict):
|
||
content: str
|
||
meaning: str
|
||
|
||
|
||
def register_jargon_hook_specs(registry: HookSpecRegistry) -> List[HookSpec]:
|
||
"""注册 jargon 系统内置 Hook 规格。
|
||
|
||
Args:
|
||
registry: 目标 Hook 规格注册中心。
|
||
|
||
Returns:
|
||
List[HookSpec]: 实际注册的 Hook 规格列表。
|
||
"""
|
||
|
||
return registry.register_hook_specs(
|
||
[
|
||
HookSpec(
|
||
name="jargon.query.before_search",
|
||
description="Maisaka 黑话查询工具执行检索前触发，可改写词条列表、检索参数或直接中止。",
|
||
parameters_schema=build_object_schema(
|
||
{
|
||
"words": {
|
||
"type": "array",
|
||
"items": {"type": "string"},
|
||
"description": "准备查询的黑话词条列表。",
|
||
},
|
||
"session_id": {"type": "string", "description": "当前会话 ID。"},
|
||
"limit": {"type": "integer", "description": "单个词条的最大返回条数。"},
|
||
"case_sensitive": {"type": "boolean", "description": "是否大小写敏感。"},
|
||
"enable_fuzzy_fallback": {"type": "boolean", "description": "是否允许精确命中失败后回退模糊检索。"},
|
||
"abort_message": {"type": "string", "description": "Hook 主动中止时的失败提示。"},
|
||
},
|
||
required=["words", "session_id", "limit", "case_sensitive", "enable_fuzzy_fallback"],
|
||
),
|
||
default_timeout_ms=5000,
|
||
allow_abort=True,
|
||
allow_kwargs_mutation=True,
|
||
),
|
||
HookSpec(
|
||
name="jargon.query.after_search",
|
||
description="Maisaka 黑话查询工具完成检索后触发，可改写结果列表或中止返回。",
|
||
parameters_schema=build_object_schema(
|
||
{
|
||
"words": {
|
||
"type": "array",
|
||
"items": {"type": "string"},
|
||
"description": "实际查询的黑话词条列表。",
|
||
},
|
||
"session_id": {"type": "string", "description": "当前会话 ID。"},
|
||
"limit": {"type": "integer", "description": "单个词条的最大返回条数。"},
|
||
"case_sensitive": {"type": "boolean", "description": "是否大小写敏感。"},
|
||
"enable_fuzzy_fallback": {"type": "boolean", "description": "是否启用了模糊检索回退。"},
|
||
"results": {
|
||
"type": "array",
|
||
"items": {"type": "object"},
|
||
"description": "查询结果列表。",
|
||
},
|
||
"abort_message": {"type": "string", "description": "Hook 主动中止时的失败提示。"},
|
||
},
|
||
required=["words", "session_id", "limit", "case_sensitive", "enable_fuzzy_fallback", "results"],
|
||
),
|
||
default_timeout_ms=5000,
|
||
allow_abort=True,
|
||
allow_kwargs_mutation=True,
|
||
),
|
||
HookSpec(
|
||
name="jargon.extract.before_persist",
|
||
description="黑话条目准备写入数据库前触发，可改写去重后的条目列表或跳过本次持久化。",
|
||
parameters_schema=build_object_schema(
|
||
{
|
||
"session_id": {"type": "string", "description": "当前会话 ID。"},
|
||
"session_name": {"type": "string", "description": "当前会话展示名称。"},
|
||
"entries": {
|
||
"type": "array",
|
||
"items": {"type": "object"},
|
||
"description": "即将持久化的黑话条目列表。",
|
||
},
|
||
},
|
||
required=["session_id", "session_name", "entries"],
|
||
),
|
||
default_timeout_ms=5000,
|
||
allow_abort=True,
|
||
allow_kwargs_mutation=True,
|
||
),
|
||
HookSpec(
|
||
name="jargon.inference.before_finalize",
|
||
description="黑话含义推断完成、写回数据库前触发，可改写最终判定与含义结果。",
|
||
parameters_schema=build_object_schema(
|
||
{
|
||
"session_id": {"type": "string", "description": "当前会话 ID。"},
|
||
"session_name": {"type": "string", "description": "当前会话展示名称。"},
|
||
"content": {"type": "string", "description": "当前黑话词条。"},
|
||
"count": {"type": "integer", "description": "当前词条累计命中次数。"},
|
||
"raw_content_list": {
|
||
"type": "array",
|
||
"items": {"type": "string"},
|
||
"description": "用于推断的原始上下文片段列表。",
|
||
},
|
||
"inference_with_context": {"type": "object", "description": "基于上下文的推断结果。"},
|
||
"inference_with_content_only": {"type": "object", "description": "仅基于词条内容的推断结果。"},
|
||
"comparison_result": {"type": "object", "description": "比较阶段输出结果。"},
|
||
"is_jargon": {"type": "boolean", "description": "当前推断是否判定为黑话。"},
|
||
"meaning": {"type": "string", "description": "当前推断出的黑话含义。"},
|
||
"is_complete": {"type": "boolean", "description": "当前是否已完成全部推断流程。"},
|
||
"last_inference_count": {"type": "integer", "description": "本次推断完成后应写回的 last_inference_count。"},
|
||
},
|
||
required=[
|
||
"session_id",
|
||
"session_name",
|
||
"content",
|
||
"count",
|
||
"raw_content_list",
|
||
"inference_with_context",
|
||
"inference_with_content_only",
|
||
"comparison_result",
|
||
"is_jargon",
|
||
"meaning",
|
||
"is_complete",
|
||
"last_inference_count",
|
||
],
|
||
),
|
||
default_timeout_ms=5000,
|
||
allow_abort=True,
|
||
allow_kwargs_mutation=True,
|
||
),
|
||
]
|
||
)
|
||
|
||
|
||
class JargonMiner:
|
||
def __init__(self, session_id: str, session_name: str) -> None:
|
||
"""初始化黑话学习器。
|
||
|
||
Args:
|
||
session_id: 当前会话 ID。
|
||
session_name: 当前会话展示名称。
|
||
"""
|
||
|
||
self.session_id = session_id
|
||
self.session_name = session_name
|
||
|
||
# Cache 相关
|
||
self.cache_limit = 50
|
||
self.cache: OrderedDict[str, None] = OrderedDict()
|
||
# 黑话提取锁，防止并发执行
|
||
self._extraction_lock = asyncio.Lock()
|
||
|
||
@staticmethod
|
||
def _get_runtime_manager() -> Any:
|
||
"""获取插件运行时管理器。
|
||
|
||
Returns:
|
||
Any: 插件运行时管理器单例。
|
||
"""
|
||
|
||
from src.plugin_runtime.integration import get_plugin_runtime_manager
|
||
|
||
return get_plugin_runtime_manager()
|
||
|
||
@staticmethod
|
||
def _coerce_int(value: Any, default: int) -> int:
|
||
"""将任意值安全转换为整数。
|
||
|
||
Args:
|
||
value: 待转换的值。
|
||
default: 转换失败时使用的默认值。
|
||
|
||
Returns:
|
||
int: 转换后的整数结果。
|
||
"""
|
||
|
||
try:
|
||
return int(value)
|
||
except (TypeError, ValueError):
|
||
return default
|
||
|
||
@staticmethod
|
||
def _serialize_jargon_entries(entries: List[JargonEntry]) -> List[Dict[str, object]]:
|
||
"""将黑话条目列表序列化为 Hook 可传输结构。
|
||
|
||
Args:
|
||
entries: 原始黑话条目列表。
|
||
|
||
Returns:
|
||
List[Dict[str, object]]: 序列化后的条目列表。
|
||
"""
|
||
|
||
return [
|
||
{
|
||
"content": str(entry["content"]).strip(),
|
||
"raw_content": sorted(str(item).strip() for item in entry["raw_content"] if str(item).strip()),
|
||
}
|
||
for entry in entries
|
||
if str(entry["content"]).strip()
|
||
]
|
||
|
||
@staticmethod
|
||
def _deserialize_jargon_entries(raw_entries: Any) -> List[JargonEntry]:
|
||
"""从 Hook 载荷恢复黑话条目列表。
|
||
|
||
Args:
|
||
raw_entries: Hook 返回的条目数据。
|
||
|
||
Returns:
|
||
List[JargonEntry]: 恢复后的黑话条目列表。
|
||
"""
|
||
|
||
if not isinstance(raw_entries, list):
|
||
return []
|
||
|
||
normalized_entries: List[JargonEntry] = []
|
||
for raw_entry in raw_entries:
|
||
if not isinstance(raw_entry, dict):
|
||
continue
|
||
content = str(raw_entry.get("content") or "").strip()
|
||
if not content:
|
||
continue
|
||
raw_content_values = raw_entry.get("raw_content")
|
||
raw_content: Set[str] = set()
|
||
if isinstance(raw_content_values, list):
|
||
raw_content = {str(item).strip() for item in raw_content_values if str(item).strip()}
|
||
normalized_entries.append({"content": content, "raw_content": raw_content})
|
||
return normalized_entries
|
||
|
||
def get_cached_jargons(self) -> List[str]:
|
||
"""获取缓存中的所有黑话列表"""
|
||
return list(self.cache.keys())
|
||
|
||
async def infer_meaning(self, jargon_obj: MaiJargon) -> None:
|
||
"""对黑话条目执行含义推断。
|
||
|
||
Args:
|
||
jargon_obj: 待推断的黑话数据对象。
|
||
"""
|
||
content = jargon_obj.content
|
||
# 解析raw_content列表
|
||
raw_content_list = []
|
||
if raw_content_str := jargon_obj.raw_content:
|
||
try:
|
||
raw_content_list = json.loads(raw_content_str)
|
||
if not isinstance(raw_content_list, list):
|
||
raw_content_list = [raw_content_list] if raw_content_list else []
|
||
except (json.JSONDecodeError, TypeError):
|
||
raw_content_list = [raw_content_str] if raw_content_str else []
|
||
|
||
if not raw_content_list:
|
||
logger.warning(f"jargon {content} 没有raw_content，跳过推断")
|
||
return
|
||
|
||
# 获取当前count和上一次的meaning
|
||
current_count = jargon_obj.count
|
||
previous_meaning = jargon_obj.meaning
|
||
|
||
# 步骤1: 基于raw_content和content推断
|
||
raw_content_text = "\n".join(raw_content_list)
|
||
|
||
# 当count为24, 60时，随机移除一半的raw_content项目
|
||
if current_count in [24, 60] and len(raw_content_list) > 1:
|
||
# 计算要保留的数量（至少保留1个）
|
||
keep_count = max(1, len(raw_content_list) // 2)
|
||
raw_content_list = random.sample(raw_content_list, keep_count)
|
||
logger.info(
|
||
f"jargon {content} count={current_count}，随机移除后剩余 {len(raw_content_list)} 个raw_content项目"
|
||
)
|
||
|
||
# 当count为24, 60, 100时，在prompt中放入上一次推断出的meaning作为参考
|
||
previous_meaning_section = ""
|
||
previous_meaning_instruction = ""
|
||
if current_count in [24, 60, 100] and previous_meaning:
|
||
previous_meaning_section = f"\n**上一次推断的含义（仅供参考）**\n{previous_meaning}"
|
||
previous_meaning_instruction = "- 请参考上一次推断的含义，结合新的上下文信息，给出更准确或更新的推断结果"
|
||
|
||
prompt1_template = prompt_manager.get_prompt("jargon_inference_with_context")
|
||
prompt1_template.add_context("bot_name", global_config.bot.nickname)
|
||
prompt1_template.add_context("content", str(content))
|
||
prompt1_template.add_context("raw_content_list", raw_content_text)
|
||
prompt1_template.add_context("previous_meaning_section", previous_meaning_section)
|
||
prompt1_template.add_context("previous_meaning_instruction", previous_meaning_instruction)
|
||
prompt1 = await prompt_manager.render_prompt(prompt1_template)
|
||
|
||
generation_result_1 = await llm_inference.generate_response(
|
||
prompt1, options=LLMGenerationOptions(temperature=0.3)
|
||
)
|
||
llm_response_1 = generation_result_1.response
|
||
if not llm_response_1:
|
||
logger.warning(f"jargon {content} 推断1失败：无响应")
|
||
return
|
||
|
||
# 解析推断1结果
|
||
inference1 = self._parse_result(llm_response_1)
|
||
if not inference1:
|
||
logger.warning(f"jargon {content} 推断1解析失败")
|
||
return
|
||
|
||
no_info = inference1.get("no_info", False)
|
||
meaning1: str = inference1.get("meaning", "").strip()
|
||
if no_info or not meaning1:
|
||
logger.info(f"jargon {content} 推断1表示信息不足无法推断，放弃本次推断，待下次更新")
|
||
# 更新最后一次判定的count值，避免在同一阈值重复尝试
|
||
jargon_obj.last_inference_count = jargon_obj.count or 0
|
||
|
||
try:
|
||
self._modify_jargon_entry(jargon_obj)
|
||
except Exception as e:
|
||
logger.error(f"jargon {content} 推断1更新last_inference_count失败: {e}")
|
||
return
|
||
|
||
# 步骤2: 基于content-only进行推断
|
||
prompt2_template = prompt_manager.get_prompt("jargon_inference_content_only")
|
||
prompt2_template.add_context("content", content)
|
||
prompt2 = await prompt_manager.render_prompt(prompt2_template)
|
||
|
||
generation_result_2 = await llm_inference.generate_response(
|
||
prompt2, options=LLMGenerationOptions(temperature=0.3)
|
||
)
|
||
llm_response_2 = generation_result_2.response
|
||
if not llm_response_2:
|
||
logger.warning(f"jargon {content} 推断2失败：无响应")
|
||
return
|
||
|
||
# 解析推断2结果
|
||
inference2 = self._parse_result(llm_response_2)
|
||
if not inference2:
|
||
logger.warning(f"jargon {content} 推断2解析失败")
|
||
return
|
||
|
||
if global_config.debug.show_jargon_prompt:
|
||
logger.info(f"jargon {content} 推断1提示词: {prompt1}")
|
||
logger.info(f"jargon {content} 推断2提示词: {prompt2}")
|
||
|
||
# 步骤3: 比较两个推断结果
|
||
prompt3_template = prompt_manager.get_prompt("jargon_compare_inference")
|
||
prompt3_template.add_context("inference1", json.dumps(inference1, ensure_ascii=False))
|
||
prompt3_template.add_context("inference2", json.dumps(inference2, ensure_ascii=False))
|
||
prompt3 = await prompt_manager.render_prompt(prompt3_template)
|
||
|
||
if global_config.debug.show_jargon_prompt:
|
||
logger.info(f"jargon {content} 比较提示词: {prompt3}")
|
||
|
||
generation_result_3 = await llm_inference.generate_response(
|
||
prompt3, options=LLMGenerationOptions(temperature=0.3)
|
||
)
|
||
llm_response_3 = generation_result_3.response
|
||
if not llm_response_3:
|
||
logger.warning(f"jargon {content} 比较失败：无响应")
|
||
return
|
||
|
||
comparison_result = self._parse_result(llm_response_3)
|
||
if not comparison_result:
|
||
logger.warning(f"jargon {content} 比较解析失败")
|
||
return
|
||
|
||
is_similar = comparison_result.get("is_similar", False)
|
||
is_jargon = not is_similar # 如果相似，说明不是黑话；如果有差异，说明是黑话
|
||
|
||
finalized_meaning = inference1.get("meaning", "") if is_jargon else ""
|
||
is_complete = (jargon_obj.count or 0) >= 100
|
||
last_inference_count = jargon_obj.count or 0
|
||
finalize_result = await self._get_runtime_manager().invoke_hook(
|
||
"jargon.inference.before_finalize",
|
||
session_id=self.session_id,
|
||
session_name=self.session_name,
|
||
content=content,
|
||
count=current_count,
|
||
raw_content_list=list(raw_content_list),
|
||
inference_with_context=dict(inference1),
|
||
inference_with_content_only=dict(inference2),
|
||
comparison_result=dict(comparison_result),
|
||
is_jargon=is_jargon,
|
||
meaning=finalized_meaning,
|
||
is_complete=is_complete,
|
||
last_inference_count=last_inference_count,
|
||
)
|
||
if finalize_result.aborted:
|
||
logger.info(f"jargon {content} 的推断结果被 Hook 中止写回")
|
||
return
|
||
|
||
finalize_kwargs = finalize_result.kwargs
|
||
is_jargon = bool(finalize_kwargs.get("is_jargon", is_jargon))
|
||
finalized_meaning = str(finalize_kwargs.get("meaning", finalized_meaning) or "").strip() if is_jargon else ""
|
||
is_complete = bool(finalize_kwargs.get("is_complete", is_complete))
|
||
last_inference_count = self._coerce_int(
|
||
finalize_kwargs.get("last_inference_count"),
|
||
last_inference_count,
|
||
)
|
||
|
||
# 更新数据库记录
|
||
jargon_obj.is_jargon = is_jargon
|
||
jargon_obj.meaning = finalized_meaning
|
||
# 更新最后一次判定的count值，避免重启后重复判定
|
||
jargon_obj.last_inference_count = last_inference_count
|
||
|
||
# 如果count>=100，标记为完成，不再进行推断
|
||
jargon_obj.is_complete = is_complete
|
||
|
||
try:
|
||
self._modify_jargon_entry(jargon_obj)
|
||
except Exception as e:
|
||
logger.error(f"jargon {content} 推断结果更新失败: {e}")
|
||
logger.debug(
|
||
f"jargon {content} 推断完成: is_jargon={is_jargon}, meaning={jargon_obj.meaning}, last_inference_count={jargon_obj.last_inference_count}, is_complete={jargon_obj.is_complete}"
|
||
)
|
||
|
||
# 固定输出推断结果，格式化为可读形式
|
||
if is_jargon:
|
||
# 是黑话，输出格式：[聊天名]xxx的含义是 xxxxxxxxxxx
|
||
meaning = jargon_obj.meaning or "无详细说明"
|
||
is_global = jargon_obj.is_global # 是否为全局的
|
||
if is_global:
|
||
logger.info(f"[黑话]{content}的含义是 {meaning}")
|
||
else:
|
||
logger.info(f"[{self.session_name}]{content}的含义是 {meaning}")
|
||
else:
|
||
# 不是黑话，输出格式：[聊天名]xxx 不是黑话
|
||
logger.info(f"[{self.session_name}]{content} 不是黑话")
|
||
|
||
async def process_extracted_entries(
|
||
self, entries: List[JargonEntry], person_name_filter: Optional[Callable[[str], bool]] = None
|
||
) -> None:
|
||
"""
|
||
处理已提取的黑话条目（从 expression_learner 路由过来的）
|
||
|
||
Args:
|
||
entries: 黑话条目列表
|
||
person_name_filter: 可选的过滤函数，用于检查内容是否包含人物名称
|
||
"""
|
||
if not entries:
|
||
return
|
||
merged_entries: Dict[str, JargonEntry] = {}
|
||
for entry in entries:
|
||
content = entry["content"].strip()
|
||
|
||
if person_name_filter and person_name_filter(content):
|
||
logger.info(f"条目 '{content}' 包含人物名称，已过滤")
|
||
continue
|
||
raw_list = entry["raw_content"] or set()
|
||
if content in merged_entries:
|
||
merged_entries[content]["raw_content"].update(raw_list)
|
||
else:
|
||
merged_entries[content] = {"content": content, "raw_content": set(raw_list)}
|
||
|
||
uniq_entries: List[JargonEntry] = list(merged_entries.values())
|
||
before_persist_result = await self._get_runtime_manager().invoke_hook(
|
||
"jargon.extract.before_persist",
|
||
session_id=self.session_id,
|
||
session_name=self.session_name,
|
||
entries=self._serialize_jargon_entries(uniq_entries),
|
||
)
|
||
if before_persist_result.aborted:
|
||
logger.info(f"[{self.session_name}] 黑话提取结果被 Hook 中止，不写入数据库")
|
||
return
|
||
|
||
raw_hook_entries = before_persist_result.kwargs.get("entries")
|
||
if raw_hook_entries is not None:
|
||
uniq_entries = self._deserialize_jargon_entries(raw_hook_entries)
|
||
if not uniq_entries:
|
||
logger.info(f"[{self.session_name}] Hook 过滤后没有可写入的黑话条目")
|
||
return
|
||
|
||
saved = 0
|
||
updated = 0
|
||
for entry in uniq_entries:
|
||
content = entry["content"]
|
||
raw_content_set = entry["raw_content"]
|
||
try:
|
||
with get_db_session(auto_commit=False) as session:
|
||
jargon_items = session.exec(select(Jargon).filter_by(content=content)).all()
|
||
except Exception as e:
|
||
logger.error(f"查询黑话 '{content}' 失败: {e}")
|
||
continue
|
||
# 找匹配项
|
||
matched_jargon: Optional[Jargon] = None
|
||
for item in jargon_items:
|
||
if global_config.expression.all_global_jargon:
|
||
# 开启all_global：所有content匹配的记录都可以
|
||
matched_jargon = item
|
||
break
|
||
else:
|
||
# 检查列表是否包含目标session_id
|
||
if item.session_id_dict:
|
||
try:
|
||
session_id_dict = json.loads(item.session_id_dict)
|
||
if self.session_id in session_id_dict:
|
||
matched_jargon = item
|
||
break
|
||
except Exception as e:
|
||
logger.error(f"解析Jargon id={item.id} session_id_list失败: {e}")
|
||
continue
|
||
if matched_jargon:
|
||
# 已存在记录，更新count和raw_content
|
||
self._update_jargon(matched_jargon, raw_content_set)
|
||
if self._should_infer_meaning(matched_jargon):
|
||
asyncio.create_task(self._infer_meaning_by_id(matched_jargon.id)) # type: ignore
|
||
updated += 1
|
||
else:
|
||
# 没找到匹配记录，创建新记录
|
||
is_global_new = global_config.expression.all_global_jargon
|
||
session_dict_str = json.dumps({self.session_id: 1})
|
||
new_jargon = Jargon(
|
||
content=content,
|
||
raw_content=json.dumps(list(raw_content_set), ensure_ascii=False),
|
||
session_id_dict=session_dict_str,
|
||
is_global=is_global_new,
|
||
count=1,
|
||
meaning="",
|
||
)
|
||
try:
|
||
with get_db_session() as session:
|
||
session.add(new_jargon)
|
||
session.flush()
|
||
saved += 1
|
||
self._add_to_cache(content)
|
||
except Exception as e:
|
||
logger.error(f"保存新黑话 '{content}' 失败: {e}")
|
||
continue
|
||
# 固定输出提取的jargon结果，格式化为可读形式（只要有提取结果就输出）
|
||
if uniq_entries:
|
||
# 收集所有提取的jargon内容
|
||
jargon_list = [entry["content"] for entry in uniq_entries]
|
||
jargon_str = ",".join(jargon_list)
|
||
logger.info(f"[{self.session_name}]疑似黑话: {jargon_str}")
|
||
|
||
if saved or updated:
|
||
logger.debug(f"jargon写入: 新增 {saved} 条，更新 {updated} 条，session_id={self.session_id}")
|
||
|
||
def _add_to_cache(self, content: str):
|
||
"""将黑话内容添加到缓存，并维护缓存大小"""
|
||
content = content.strip()
|
||
if is_single_char_jargon(content):
|
||
return
|
||
if content in self.cache:
|
||
# 已存在，移动到末尾表示最近使用
|
||
self.cache.move_to_end(content)
|
||
else:
|
||
# 新内容，添加到缓存
|
||
self.cache[content] = None
|
||
# 如果超过限制，移除最旧的项
|
||
if len(self.cache) > self.cache_limit:
|
||
removed_content, _ = self.cache.popitem(last=False)
|
||
logger.debug(f"缓存已满，移除最旧的黑话: {removed_content}")
|
||
|
||
def _update_jargon(self, db_jargon: Jargon, raw_content_set: Set[str]) -> None:
|
||
"""更新已有黑话记录并写回数据库。
|
||
|
||
Args:
|
||
db_jargon: 已命中的黑话 ORM 对象。
|
||
raw_content_set: 本次新增的原始上下文集合。
|
||
"""
|
||
db_jargon.count += 1
|
||
existing_raw_content: List[str] = []
|
||
if db_jargon.raw_content:
|
||
try:
|
||
existing_raw_content = json.loads(db_jargon.raw_content)
|
||
except Exception:
|
||
existing_raw_content = []
|
||
|
||
# 合并去重
|
||
merged_list = list(set(existing_raw_content).union(raw_content_set))
|
||
db_jargon.raw_content = json.dumps(merged_list, ensure_ascii=False)
|
||
session_id_dict: Dict[str, int] = json.loads(db_jargon.session_id_dict)
|
||
session_id_dict[self.session_id] = session_id_dict.get(self.session_id, 0) + 1
|
||
db_jargon.session_id_dict = json.dumps(session_id_dict)
|
||
|
||
# 开启all_global时，确保记录标记为is_global=True
|
||
if global_config.expression.all_global_jargon:
|
||
db_jargon.is_global = True
|
||
|
||
try:
|
||
with get_db_session() as session:
|
||
if db_jargon.id is None:
|
||
raise ValueError("黑话记录缺少 id，无法更新数据库")
|
||
statement = select(Jargon).filter_by(id=db_jargon.id).limit(1)
|
||
if persisted_jargon := session.exec(statement).first():
|
||
persisted_jargon.count = db_jargon.count
|
||
persisted_jargon.raw_content = db_jargon.raw_content
|
||
persisted_jargon.session_id_dict = db_jargon.session_id_dict
|
||
persisted_jargon.is_global = db_jargon.is_global
|
||
session.add(persisted_jargon)
|
||
else:
|
||
logger.warning(f"黑话 ID {db_jargon.id} 在数据库中未找到，无法更新")
|
||
except Exception as e:
|
||
logger.error(f"更新黑话 '{db_jargon.content}' 失败: {e}")
|
||
|
||
def _parse_result(self, response: str) -> Optional[Dict[str, str]]:
|
||
try:
|
||
result = json.loads(response.strip())
|
||
except Exception:
|
||
try:
|
||
repaired = repair_json(response.strip())
|
||
result = json.loads(repaired)
|
||
except Exception as e2:
|
||
logger.error(f"推断结果解析失败: {e2}")
|
||
return None
|
||
if not isinstance(result, dict):
|
||
logger.warning("推断结果格式错误")
|
||
return None
|
||
return result
|
||
|
||
def _modify_jargon_entry(self, jargon_obj: MaiJargon) -> None:
|
||
with get_db_session() as session:
|
||
if not jargon_obj.item_id:
|
||
raise ValueError("jargon_obj must have item_id to update")
|
||
statement = select(Jargon).filter_by(id=jargon_obj.item_id).limit(1)
|
||
if db_record := session.exec(statement).first():
|
||
db_record.is_jargon = jargon_obj.is_jargon
|
||
db_record.meaning = jargon_obj.meaning
|
||
db_record.last_inference_count = jargon_obj.last_inference_count
|
||
db_record.is_complete = jargon_obj.is_complete
|
||
session.add(db_record)
|
||
|
||
def _should_infer_meaning(self, jargon_obj: Jargon) -> bool:
|
||
"""
|
||
判断是否需要进行含义推断
|
||
在 count 达到 3,6, 10, 20, 40, 60, 100 时进行推断
|
||
并且count必须大于last_inference_count，避免重启后重复判定
|
||
如果is_complete为True，不再进行推断
|
||
"""
|
||
# 如果已完成所有推断，不再推断
|
||
if jargon_obj.is_complete:
|
||
return False
|
||
|
||
count = jargon_obj.count or 0
|
||
last_inference = jargon_obj.last_inference_count or 0
|
||
|
||
# 阈值列表：3,6, 10, 20, 40, 60, 100
|
||
thresholds = [2, 4, 8, 12, 24, 60, 100]
|
||
|
||
if count < thresholds[0]:
|
||
return False
|
||
# 如果count没有超过上次判定值，不需要判定
|
||
if count <= last_inference:
|
||
return False
|
||
|
||
next_threshold = next(
|
||
(threshold for threshold in thresholds if threshold > last_inference),
|
||
None,
|
||
)
|
||
# 如果没有找到下一个阈值，说明已经超过100，不应该再推断
|
||
return False if next_threshold is None else count >= next_threshold
|
||
|
||
async def _infer_meaning_by_id(self, jargon_id: int):
|
||
jargon_obj: Optional[MaiJargon] = None
|
||
try:
|
||
with get_db_session() as session:
|
||
statement = select(Jargon).filter_by(id=jargon_id).limit(1)
|
||
if db_record := session.exec(statement).first():
|
||
jargon_obj = MaiJargon.from_db_instance(db_record)
|
||
except Exception as e:
|
||
logger.error(f"查询Jargon id={jargon_id}失败: {e}")
|
||
return
|
||
if jargon_obj:
|
||
await self.infer_meaning(jargon_obj)
|