添加 A_Memorix 插件 v2.0.0(包含运行时与文档)

引入 A_Memorix 插件 v2.0.0:新增大量运行时组件、存储/模式更新、检索能力提升、管理工具、导入/调优工作流以及相关文档。关键新增内容包括:lifecycle_orchestrator、SDKMemoryKernel/运行时初始化器、新的存储层与 metadata_store 变更(SCHEMA_VERSION v8)、检索增强(双路径检索、图关系召回、稀疏 BM25),以及多种工具服务(episode/person_profile/relation/segmentation/tuning/search execution)。同时新增 Web 导入/摘要导入器及大量维护脚本。还更新了插件清单、embedding API 适配器、plugin.py、requirements/pyproject,以及主入口文件,使新插件接入项目。该变更为 2.0.0 版本发布做好准备,实现统一的 SDK Tool 接口并扩展整体运行能力。
This commit is contained in:
DawnARC
2026-03-19 00:09:04 +08:00
parent eb257345dd
commit 71b3a828c6
44 changed files with 18193 additions and 405 deletions

View File

@@ -1,8 +1,16 @@
"""SDK runtime exports for A_Memorix."""
from .search_runtime_initializer import (
SearchRuntimeBundle,
SearchRuntimeInitializer,
build_search_runtime,
)
from .sdk_memory_kernel import KernelSearchRequest, SDKMemoryKernel
__all__ = [
"SearchRuntimeBundle",
"SearchRuntimeInitializer",
"build_search_runtime",
"KernelSearchRequest",
"SDKMemoryKernel",
]

View File

@@ -0,0 +1,268 @@
"""Lifecycle bootstrap/teardown helpers extracted from plugin.py."""
from __future__ import annotations
import asyncio
from pathlib import Path
from typing import Any
from src.common.logger import get_logger
from ..embedding import create_embedding_api_adapter
from ..retrieval import SparseBM25Config, SparseBM25Index
from ..storage import (
GraphStore,
MetadataStore,
QuantizationType,
SparseMatrixFormat,
VectorStore,
)
from ..utils.runtime_self_check import ensure_runtime_self_check
from ..utils.relation_write_service import RelationWriteService
logger = get_logger("A_Memorix.LifecycleOrchestrator")
async def ensure_initialized(plugin: Any) -> None:
if plugin._initialized:
plugin._runtime_ready = plugin._check_storage_ready()
return
async with plugin._init_lock:
if plugin._initialized:
plugin._runtime_ready = plugin._check_storage_ready()
return
logger.info("A_Memorix 插件正在异步初始化存储组件...")
plugin._validate_runtime_config()
await initialize_storage_async(plugin)
report = await ensure_runtime_self_check(plugin, force=True)
if not bool(report.get("ok", False)):
logger.error(
"A_Memorix runtime self-check failed: "
f"{report.get('message', 'unknown')}; "
"建议执行 python plugins/A_memorix/scripts/runtime_self_check.py --json"
)
if plugin.graph_store and plugin.metadata_store:
relation_count = plugin.metadata_store.count_relations()
if relation_count > 0 and not plugin.graph_store.has_edge_hash_map():
raise RuntimeError(
"检测到 relations 数据存在但 edge-hash-map 为空。"
" 请先执行 scripts/release_vnext_migrate.py migrate。"
)
plugin._initialized = True
plugin._runtime_ready = plugin._check_storage_ready()
plugin._update_plugin_config()
logger.info("A_Memorix 插件异步初始化成功")
def start_background_tasks(plugin: Any) -> None:
"""Start background tasks idempotently."""
if not hasattr(plugin, "_episode_generation_task"):
plugin._episode_generation_task = None
if (
plugin.get_config("summarization.enabled", True)
and plugin.get_config("schedule.enabled", True)
and (plugin._scheduled_import_task is None or plugin._scheduled_import_task.done())
):
plugin._scheduled_import_task = asyncio.create_task(plugin._scheduled_import_loop())
if (
plugin.get_config("advanced.enable_auto_save", True)
and (plugin._auto_save_task is None or plugin._auto_save_task.done())
):
plugin._auto_save_task = asyncio.create_task(plugin._auto_save_loop())
if (
plugin.get_config("person_profile.enabled", True)
and (plugin._person_profile_refresh_task is None or plugin._person_profile_refresh_task.done())
):
plugin._person_profile_refresh_task = asyncio.create_task(plugin._person_profile_refresh_loop())
if plugin._memory_maintenance_task is None or plugin._memory_maintenance_task.done():
plugin._memory_maintenance_task = asyncio.create_task(plugin._memory_maintenance_loop())
rv_cfg = plugin.get_config("retrieval.relation_vectorization", {}) or {}
if isinstance(rv_cfg, dict):
rv_enabled = bool(rv_cfg.get("enabled", False))
rv_backfill = bool(rv_cfg.get("backfill_enabled", False))
else:
rv_enabled = False
rv_backfill = False
if rv_enabled and rv_backfill and (
plugin._relation_vector_backfill_task is None or plugin._relation_vector_backfill_task.done()
):
plugin._relation_vector_backfill_task = asyncio.create_task(plugin._relation_vector_backfill_loop())
episode_task = getattr(plugin, "_episode_generation_task", None)
episode_loop = getattr(plugin, "_episode_generation_loop", None)
if (
callable(episode_loop)
and bool(plugin.get_config("episode.enabled", True))
and bool(plugin.get_config("episode.generation_enabled", True))
and (episode_task is None or episode_task.done())
):
plugin._episode_generation_task = asyncio.create_task(episode_loop())
async def cancel_background_tasks(plugin: Any) -> None:
"""Cancel all background tasks and wait for cleanup."""
tasks = [
("scheduled_import", plugin._scheduled_import_task),
("auto_save", plugin._auto_save_task),
("person_profile_refresh", plugin._person_profile_refresh_task),
("memory_maintenance", plugin._memory_maintenance_task),
("relation_vector_backfill", plugin._relation_vector_backfill_task),
("episode_generation", getattr(plugin, "_episode_generation_task", None)),
]
for _, task in tasks:
if task and not task.done():
task.cancel()
for name, task in tasks:
if not task:
continue
try:
await task
except asyncio.CancelledError:
pass
except Exception as e:
logger.warning(f"后台任务 {name} 退出异常: {e}")
plugin._scheduled_import_task = None
plugin._auto_save_task = None
plugin._person_profile_refresh_task = None
plugin._memory_maintenance_task = None
plugin._relation_vector_backfill_task = None
plugin._episode_generation_task = None
async def initialize_storage_async(plugin: Any) -> None:
"""Initialize storage components asynchronously."""
data_dir_str = plugin.get_config("storage.data_dir", "./data")
if data_dir_str.startswith("."):
plugin_dir = Path(__file__).resolve().parents[2]
data_dir = (plugin_dir / data_dir_str).resolve()
else:
data_dir = Path(data_dir_str)
logger.info(f"A_Memorix 数据存储路径: {data_dir}")
data_dir.mkdir(parents=True, exist_ok=True)
plugin.embedding_manager = create_embedding_api_adapter(
batch_size=plugin.get_config("embedding.batch_size", 32),
max_concurrent=plugin.get_config("embedding.max_concurrent", 5),
default_dimension=plugin.get_config("embedding.dimension", 1024),
model_name=plugin.get_config("embedding.model_name", "auto"),
retry_config=plugin.get_config("embedding.retry", {}),
)
logger.info("嵌入 API 适配器初始化完成")
try:
detected_dimension = await plugin.embedding_manager._detect_dimension()
logger.info(f"嵌入维度检测成功: {detected_dimension}")
except Exception as e:
logger.warning(f"嵌入维度检测失败: {e},使用默认值")
detected_dimension = plugin.embedding_manager.default_dimension
quantization_str = plugin.get_config("embedding.quantization_type", "int8")
if str(quantization_str or "").strip().lower() != "int8":
raise ValueError("embedding.quantization_type 在 vNext 仅允许 int8(SQ8)。")
quantization_type = QuantizationType.INT8
plugin.vector_store = VectorStore(
dimension=detected_dimension,
quantization_type=quantization_type,
data_dir=data_dir / "vectors",
)
plugin.vector_store.min_train_threshold = plugin.get_config("embedding.min_train_threshold", 40)
logger.info(
"向量存储初始化完成("
f"维度: {detected_dimension}, "
f"训练阈值: {plugin.vector_store.min_train_threshold}"
)
matrix_format_str = plugin.get_config("graph.sparse_matrix_format", "csr")
matrix_format_map = {
"csr": SparseMatrixFormat.CSR,
"csc": SparseMatrixFormat.CSC,
}
matrix_format = matrix_format_map.get(matrix_format_str, SparseMatrixFormat.CSR)
plugin.graph_store = GraphStore(
matrix_format=matrix_format,
data_dir=data_dir / "graph",
)
logger.info("图存储初始化完成")
plugin.metadata_store = MetadataStore(data_dir=data_dir / "metadata")
plugin.metadata_store.connect()
logger.info("元数据存储初始化完成")
plugin.relation_write_service = RelationWriteService(
metadata_store=plugin.metadata_store,
graph_store=plugin.graph_store,
vector_store=plugin.vector_store,
embedding_manager=plugin.embedding_manager,
)
logger.info("关系写入服务初始化完成")
sparse_cfg_raw = plugin.get_config("retrieval.sparse", {}) or {}
if not isinstance(sparse_cfg_raw, dict):
sparse_cfg_raw = {}
try:
sparse_cfg = SparseBM25Config(**sparse_cfg_raw)
except Exception as e:
logger.warning(f"sparse 配置非法,回退默认配置: {e}")
sparse_cfg = SparseBM25Config()
plugin.sparse_index = SparseBM25Index(
metadata_store=plugin.metadata_store,
config=sparse_cfg,
)
logger.info(
"稀疏检索组件初始化完成: "
f"enabled={sparse_cfg.enabled}, "
f"lazy_load={sparse_cfg.lazy_load}, "
f"mode={sparse_cfg.mode}, "
f"tokenizer={sparse_cfg.tokenizer_mode}"
)
if sparse_cfg.enabled and not sparse_cfg.lazy_load:
plugin.sparse_index.ensure_loaded()
if plugin.vector_store.has_data():
try:
plugin.vector_store.load()
logger.info(f"向量数据已加载,共 {plugin.vector_store.num_vectors} 个向量")
except Exception as e:
logger.warning(f"加载向量数据失败: {e}")
try:
warmup_summary = plugin.vector_store.warmup_index(force_train=True)
if warmup_summary.get("ok"):
logger.info(
"向量索引预热完成: "
f"trained={warmup_summary.get('trained')}, "
f"index_ntotal={warmup_summary.get('index_ntotal')}, "
f"fallback_ntotal={warmup_summary.get('fallback_ntotal')}, "
f"bin_count={warmup_summary.get('bin_count')}, "
f"duration_ms={float(warmup_summary.get('duration_ms', 0.0)):.2f}"
)
else:
logger.warning(
"向量索引预热失败,继续启用 sparse 降级路径: "
f"{warmup_summary.get('error', 'unknown')}"
)
except Exception as e:
logger.warning(f"向量索引预热异常,继续启用 sparse 降级路径: {e}")
if plugin.graph_store.has_data():
try:
plugin.graph_store.load()
logger.info(f"图数据已加载,共 {plugin.graph_store.num_nodes} 个节点")
except Exception as e:
logger.warning(f"加载图数据失败: {e}")
logger.info(f"知识库数据目录: {data_dir}")

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,240 @@
"""Shared runtime initializer for Action/Tool/Command retrieval components."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Dict, Optional
from src.common.logger import get_logger
from ..retrieval import (
DualPathRetriever,
DualPathRetrieverConfig,
DynamicThresholdFilter,
FusionConfig,
GraphRelationRecallConfig,
RelationIntentConfig,
RetrievalStrategy,
SparseBM25Config,
ThresholdConfig,
ThresholdMethod,
)
_logger = get_logger("A_Memorix.SearchRuntimeInitializer")
_REQUIRED_COMPONENT_KEYS = (
"vector_store",
"graph_store",
"metadata_store",
"embedding_manager",
)
def _get_config_value(config: Optional[dict], key: str, default: Any = None) -> Any:
if not isinstance(config, dict):
return default
current: Any = config
for part in key.split("."):
if isinstance(current, dict) and part in current:
current = current[part]
else:
return default
return current
def _safe_dict(value: Any) -> Dict[str, Any]:
return value if isinstance(value, dict) else {}
def _resolve_debug_enabled(plugin_config: Optional[dict]) -> bool:
advanced = _get_config_value(plugin_config, "advanced", {})
if isinstance(advanced, dict):
return bool(advanced.get("debug", False))
return bool(_get_config_value(plugin_config, "debug", False))
@dataclass
class SearchRuntimeBundle:
"""Resolved runtime components and initialized retriever/filter."""
vector_store: Optional[Any] = None
graph_store: Optional[Any] = None
metadata_store: Optional[Any] = None
embedding_manager: Optional[Any] = None
sparse_index: Optional[Any] = None
retriever: Optional[DualPathRetriever] = None
threshold_filter: Optional[DynamicThresholdFilter] = None
error: str = ""
@property
def ready(self) -> bool:
return (
self.retriever is not None
and self.vector_store is not None
and self.graph_store is not None
and self.metadata_store is not None
and self.embedding_manager is not None
)
def _resolve_runtime_components(plugin_config: Optional[dict]) -> SearchRuntimeBundle:
bundle = SearchRuntimeBundle(
vector_store=_get_config_value(plugin_config, "vector_store"),
graph_store=_get_config_value(plugin_config, "graph_store"),
metadata_store=_get_config_value(plugin_config, "metadata_store"),
embedding_manager=_get_config_value(plugin_config, "embedding_manager"),
sparse_index=_get_config_value(plugin_config, "sparse_index"),
)
missing_required = any(
getattr(bundle, key) is None for key in _REQUIRED_COMPONENT_KEYS
)
if not missing_required:
return bundle
try:
from ...plugin import AMemorixPlugin
instances = AMemorixPlugin.get_storage_instances()
except Exception:
instances = {}
if not isinstance(instances, dict) or not instances:
return bundle
if bundle.vector_store is None:
bundle.vector_store = instances.get("vector_store")
if bundle.graph_store is None:
bundle.graph_store = instances.get("graph_store")
if bundle.metadata_store is None:
bundle.metadata_store = instances.get("metadata_store")
if bundle.embedding_manager is None:
bundle.embedding_manager = instances.get("embedding_manager")
if bundle.sparse_index is None:
bundle.sparse_index = instances.get("sparse_index")
return bundle
def build_search_runtime(
plugin_config: Optional[dict],
logger_obj: Optional[Any],
owner_tag: str,
*,
log_prefix: str = "",
) -> SearchRuntimeBundle:
"""Build retriever + threshold filter with unified fallback/config parsing."""
log = logger_obj or _logger
owner = str(owner_tag or "runtime").strip().lower() or "runtime"
prefix = str(log_prefix or "").strip()
prefix_text = f"{prefix} " if prefix else ""
runtime = _resolve_runtime_components(plugin_config)
if any(getattr(runtime, key) is None for key in _REQUIRED_COMPONENT_KEYS):
runtime.error = "存储组件未完全初始化"
log.warning(f"{prefix_text}[{owner}] 存储组件未完全初始化,无法使用检索功能")
return runtime
sparse_cfg_raw = _safe_dict(_get_config_value(plugin_config, "retrieval.sparse", {}) or {})
fusion_cfg_raw = _safe_dict(_get_config_value(plugin_config, "retrieval.fusion", {}) or {})
relation_intent_cfg_raw = _safe_dict(
_get_config_value(plugin_config, "retrieval.search.relation_intent", {}) or {}
)
graph_recall_cfg_raw = _safe_dict(
_get_config_value(plugin_config, "retrieval.search.graph_recall", {}) or {}
)
try:
sparse_cfg = SparseBM25Config(**sparse_cfg_raw)
except Exception as e:
log.warning(f"{prefix_text}[{owner}] sparse 配置非法,回退默认: {e}")
sparse_cfg = SparseBM25Config()
try:
fusion_cfg = FusionConfig(**fusion_cfg_raw)
except Exception as e:
log.warning(f"{prefix_text}[{owner}] fusion 配置非法,回退默认: {e}")
fusion_cfg = FusionConfig()
try:
relation_intent_cfg = RelationIntentConfig(**relation_intent_cfg_raw)
except Exception as e:
log.warning(f"{prefix_text}[{owner}] relation_intent 配置非法,回退默认: {e}")
relation_intent_cfg = RelationIntentConfig()
try:
graph_recall_cfg = GraphRelationRecallConfig(**graph_recall_cfg_raw)
except Exception as e:
log.warning(f"{prefix_text}[{owner}] graph_recall 配置非法,回退默认: {e}")
graph_recall_cfg = GraphRelationRecallConfig()
try:
config = DualPathRetrieverConfig(
top_k_paragraphs=_get_config_value(plugin_config, "retrieval.top_k_paragraphs", 20),
top_k_relations=_get_config_value(plugin_config, "retrieval.top_k_relations", 10),
top_k_final=_get_config_value(plugin_config, "retrieval.top_k_final", 10),
alpha=_get_config_value(plugin_config, "retrieval.alpha", 0.5),
enable_ppr=_get_config_value(plugin_config, "retrieval.enable_ppr", True),
ppr_alpha=_get_config_value(plugin_config, "retrieval.ppr_alpha", 0.85),
ppr_timeout_seconds=_get_config_value(
plugin_config, "retrieval.ppr_timeout_seconds", 1.5
),
ppr_concurrency_limit=_get_config_value(
plugin_config, "retrieval.ppr_concurrency_limit", 4
),
enable_parallel=_get_config_value(plugin_config, "retrieval.enable_parallel", True),
retrieval_strategy=RetrievalStrategy.DUAL_PATH,
debug=_resolve_debug_enabled(plugin_config),
sparse=sparse_cfg,
fusion=fusion_cfg,
relation_intent=relation_intent_cfg,
graph_recall=graph_recall_cfg,
)
runtime.retriever = DualPathRetriever(
vector_store=runtime.vector_store,
graph_store=runtime.graph_store,
metadata_store=runtime.metadata_store,
embedding_manager=runtime.embedding_manager,
sparse_index=runtime.sparse_index,
config=config,
)
threshold_config = ThresholdConfig(
method=ThresholdMethod.ADAPTIVE,
min_threshold=_get_config_value(plugin_config, "threshold.min_threshold", 0.3),
max_threshold=_get_config_value(plugin_config, "threshold.max_threshold", 0.95),
percentile=_get_config_value(plugin_config, "threshold.percentile", 75.0),
std_multiplier=_get_config_value(plugin_config, "threshold.std_multiplier", 1.5),
min_results=_get_config_value(plugin_config, "threshold.min_results", 3),
enable_auto_adjust=_get_config_value(plugin_config, "threshold.enable_auto_adjust", True),
)
runtime.threshold_filter = DynamicThresholdFilter(threshold_config)
runtime.error = ""
log.info(f"{prefix_text}[{owner}] 检索运行时初始化完成")
except Exception as e:
runtime.retriever = None
runtime.threshold_filter = None
runtime.error = str(e)
log.error(f"{prefix_text}[{owner}] 检索运行时初始化失败: {e}")
return runtime
class SearchRuntimeInitializer:
"""Compatibility wrapper around the function style initializer."""
@staticmethod
def build_search_runtime(
plugin_config: Optional[dict],
logger_obj: Optional[Any],
owner_tag: str,
*,
log_prefix: str = "",
) -> SearchRuntimeBundle:
return build_search_runtime(
plugin_config=plugin_config,
logger_obj=logger_obj,
owner_tag=owner_tag,
log_prefix=log_prefix,
)