refactor: 将 A_Memorix 重构为主线长期记忆子系统并重建管理界面
- 将 A_Memorix 从旧 submodule / 插件形态迁入主线源码，主体落到 src/A_memorix - 调整主程序接入方式，使 A_Memorix 作为源码内长期记忆子系统运行 - 回收父项目插件体系中针对 A_Memorix 的特判，减少对 plugin 通用层的侵入 - 将长期记忆配置、运行时、自检、导入、调优等能力收口到 memory 路由与主线服务层 - 重做长期记忆控制台与图谱页面，按 MaiBot 现有 dashboard 风格接入 - 补充实体关系图与证据视图双视图能力，支持查看节点、关系、段落及其证据链路 - 新增长期记忆配置编辑器与 memory-api，支持主线内配置管理 - 补齐删除管理能力：删除预览、混合删除、来源批量删除、删除操作恢复 - 优化删除预览与删除操作详情的前端展示，支持分页、检索，并以实体名/关系内容/段落摘要替代单纯 hash 展示 - 修复图谱与控制台相关前端问题，包括证据视图切换、查询触发时机、删除弹层空值保护等 - 新增或更新 A_Memorix 相关测试、WebUI 路由测试、前端 vitest 测试与辅助验证脚本 - 移除旧 plugins/A_memorix、.gitmodules 及相关历史维护文档
This commit is contained in:
A-Dawn
4
.gitmodules
vendored
4
.gitmodules
vendored
@@ -1,4 +0,0 @@
|
||||
[submodule "plugins/A_memorix"]
|
||||
path = plugins/A_memorix
|
||||
url = https://github.com/A-Dawn/A_memorix.git
|
||||
branch = MaiBot_branch
|
||||
406
dashboard/scripts/a_memorix_electron_validate.cjs
Normal file
406
dashboard/scripts/a_memorix_electron_validate.cjs
Normal file
@@ -0,0 +1,406 @@
|
||||
const { app, BrowserWindow } = require('electron')
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
const DASHBOARD_URL = process.env.MAIBOT_DASHBOARD_URL || 'http://127.0.0.1:7999'
|
||||
const OUTPUT_DIR = process.env.MAIBOT_UI_SNAPSHOT_DIR
|
||||
|| path.resolve(__dirname, '..', '..', 'tmp', 'ui-snapshots', 'a_memorix-electron')
|
||||
const TOKEN_PATH = process.env.MAIBOT_WEBUI_TOKEN_PATH
|
||||
|| path.resolve(__dirname, '..', '..', 'data', 'webui.json')
|
||||
const sampleStamp = String(Date.now())
|
||||
const sampleSource = process.env.MAIBOT_UI_SAMPLE_SOURCE || `webui-demo:a_memorix-json-${sampleStamp}`
|
||||
const sampleName = process.env.MAIBOT_UI_SAMPLE_NAME || `webui-json-validation-${sampleStamp}.json`
|
||||
|
||||
const DEFAULT_SAMPLE = {
|
||||
paragraphs: [
|
||||
{
|
||||
content: 'Alice 在杭州西湖与 Bob 讨论 A_Memorix 的前端接入与 embedding 调优方案。',
|
||||
source: sampleSource,
|
||||
entities: ['Alice', 'Bob', '杭州西湖', 'A_Memorix'],
|
||||
relations: [
|
||||
{ subject: 'Alice', predicate: '在', object: '杭州西湖' },
|
||||
{ subject: 'Alice', predicate: '讨论', object: 'A_Memorix' },
|
||||
{ subject: 'Bob', predicate: '讨论', object: 'A_Memorix' },
|
||||
{ subject: 'Bob', predicate: '负责', object: 'embedding 调优' },
|
||||
],
|
||||
knowledge_type: 'factual',
|
||||
},
|
||||
],
|
||||
entities: ['Alice', 'Bob', '杭州西湖', 'A_Memorix', 'embedding 调优'],
|
||||
relations: [{ subject: 'Alice', predicate: '认识', object: 'Bob' }],
|
||||
}
|
||||
|
||||
function loadSampleJson() {
|
||||
const customPath = String(process.env.MAIBOT_UI_IMPORT_JSON_PATH || '').trim()
|
||||
if (!customPath) {
|
||||
return JSON.stringify(DEFAULT_SAMPLE, null, 2)
|
||||
}
|
||||
return fs.readFileSync(customPath, 'utf8')
|
||||
}
|
||||
|
||||
const sampleJson = loadSampleJson()
|
||||
|
||||
fs.mkdirSync(OUTPUT_DIR, { recursive: true })
|
||||
|
||||
function wait(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
async function exec(win, code) {
|
||||
return win.webContents.executeJavaScript(code, true)
|
||||
}
|
||||
|
||||
async function waitFor(win, predicateCode, label, timeout = 30000, interval = 300) {
|
||||
const start = Date.now()
|
||||
while (Date.now() - start < timeout) {
|
||||
try {
|
||||
const ok = await exec(win, predicateCode)
|
||||
if (ok) {
|
||||
return ok
|
||||
}
|
||||
} catch {
|
||||
// keep polling
|
||||
}
|
||||
await wait(interval)
|
||||
}
|
||||
throw new Error(`Timeout waiting for ${label}`)
|
||||
}
|
||||
|
||||
async function sendClick(win, x, y) {
|
||||
win.webContents.sendInputEvent({ type: 'mouseMove', x, y, movementX: 0, movementY: 0 })
|
||||
win.webContents.sendInputEvent({ type: 'mouseDown', x, y, button: 'left', clickCount: 1 })
|
||||
win.webContents.sendInputEvent({ type: 'mouseUp', x, y, button: 'left', clickCount: 1 })
|
||||
}
|
||||
|
||||
async function capture(win, name) {
|
||||
const image = await win.webContents.capturePage()
|
||||
fs.writeFileSync(path.join(OUTPUT_DIR, name), image.toPNG())
|
||||
const text = await exec(win, 'document.body ? document.body.innerText : ""')
|
||||
fs.writeFileSync(path.join(OUTPUT_DIR, name.replace(/\.png$/, '.txt')), text || '')
|
||||
}
|
||||
|
||||
async function getJson(win, relativePath) {
|
||||
return exec(
|
||||
win,
|
||||
`fetch(${JSON.stringify(relativePath)}, { credentials: 'include' }).then((r) => r.json())`,
|
||||
)
|
||||
}
|
||||
|
||||
async function setSessionCookie(win) {
|
||||
const raw = fs.readFileSync(TOKEN_PATH, 'utf8')
|
||||
const config = JSON.parse(raw)
|
||||
const token = String(config.access_token || '').trim()
|
||||
if (!token) {
|
||||
throw new Error(`No access token found in ${TOKEN_PATH}`)
|
||||
}
|
||||
const payload = await exec(
|
||||
win,
|
||||
`fetch('/api/webui/auth/verify', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
credentials: 'include',
|
||||
body: JSON.stringify({ token: ${JSON.stringify(token)} }),
|
||||
}).then(async (response) => ({
|
||||
ok: response.ok,
|
||||
status: response.status,
|
||||
body: await response.json(),
|
||||
}))`,
|
||||
)
|
||||
if (!payload?.ok || !payload?.body?.valid) {
|
||||
throw new Error(`Failed to authenticate WebUI token via /auth/verify: ${JSON.stringify(payload)}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function openImportTab(win) {
|
||||
await exec(win, `(() => {
|
||||
const tab = Array.from(document.querySelectorAll('[role="tab"]')).find((el) => (el.textContent || '').trim() === '导入')
|
||||
if (!tab) return false
|
||||
tab.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true, cancelable: true, pointerId: 1, button: 0, pointerType: 'mouse', isPrimary: true }))
|
||||
tab.dispatchEvent(new MouseEvent('mousedown', { bubbles: true, cancelable: true, button: 0 }))
|
||||
tab.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, cancelable: true, button: 0 }))
|
||||
tab.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, button: 0 }))
|
||||
return true
|
||||
})()`)
|
||||
await waitFor(
|
||||
win,
|
||||
`document.body && document.body.innerText.includes('粘贴导入') && document.body.innerText.includes('创建导入任务')`,
|
||||
'import panel',
|
||||
)
|
||||
}
|
||||
|
||||
async function setJsonMode(win) {
|
||||
const trigger = await exec(win, `(() => {
|
||||
const label = Array.from(document.querySelectorAll('label')).find((node) => (node.textContent || '').includes('输入模式'))
|
||||
const root = label?.closest('div')?.parentElement || label?.parentElement
|
||||
const button = root?.querySelector('button')
|
||||
if (!button) return null
|
||||
const rect = button.getBoundingClientRect()
|
||||
return { x: Math.round(rect.left + rect.width / 2), y: Math.round(rect.top + rect.height / 2) }
|
||||
})()`)
|
||||
if (!trigger) {
|
||||
throw new Error('select trigger not found')
|
||||
}
|
||||
await sendClick(win, trigger.x, trigger.y)
|
||||
await waitFor(win, `document.querySelectorAll('[role="option"]').length > 0`, 'select options', 5000, 200)
|
||||
|
||||
const option = await exec(win, `(() => {
|
||||
const item = Array.from(document.querySelectorAll('[role="option"]')).find((el) => (el.textContent || '').trim() === 'json')
|
||||
if (!item) return null
|
||||
const rect = item.getBoundingClientRect()
|
||||
return { x: Math.round(rect.left + rect.width / 2), y: Math.round(rect.top + rect.height / 2) }
|
||||
})()`)
|
||||
if (!option) {
|
||||
throw new Error('json option not found')
|
||||
}
|
||||
await sendClick(win, option.x, option.y)
|
||||
await waitFor(
|
||||
win,
|
||||
`(() => {
|
||||
const label = Array.from(document.querySelectorAll('label')).find((node) => (node.textContent || '').includes('输入模式'))
|
||||
const root = label?.closest('div')?.parentElement || label?.parentElement
|
||||
const button = root?.querySelector('button')
|
||||
return (button?.textContent || '').trim() === 'json'
|
||||
})()`,
|
||||
'json mode selected',
|
||||
8000,
|
||||
300,
|
||||
)
|
||||
}
|
||||
|
||||
async function typeIntoLabeled(win, labelText, selector, text) {
|
||||
const rect = await exec(win, `(() => {
|
||||
const label = Array.from(document.querySelectorAll('label')).find((node) => (node.textContent || '').includes(${JSON.stringify(labelText)}))
|
||||
const root = label?.closest('div')?.parentElement || label?.parentElement
|
||||
const el = root?.querySelector(${JSON.stringify(selector)})
|
||||
if (!el) return null
|
||||
const r = el.getBoundingClientRect()
|
||||
return { x: Math.round(r.left + 20), y: Math.round(r.top + 20) }
|
||||
})()`)
|
||||
if (!rect) {
|
||||
throw new Error(`field not found: ${labelText}`)
|
||||
}
|
||||
await sendClick(win, rect.x, rect.y)
|
||||
await wait(150)
|
||||
await win.webContents.insertText(text)
|
||||
await wait(250)
|
||||
}
|
||||
|
||||
async function clickButton(win, text) {
|
||||
const ok = await exec(win, `(() => {
|
||||
const target = Array.from(document.querySelectorAll('button')).find((el) => (el.textContent || '').includes(${JSON.stringify(text)}))
|
||||
if (!target) return false
|
||||
target.scrollIntoView({ block: 'center' })
|
||||
target.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true, cancelable: true, pointerId: 1, button: 0, pointerType: 'mouse', isPrimary: true }))
|
||||
target.dispatchEvent(new MouseEvent('mousedown', { bubbles: true, cancelable: true, button: 0 }))
|
||||
target.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, cancelable: true, button: 0 }))
|
||||
target.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, button: 0 }))
|
||||
return true
|
||||
})()`)
|
||||
if (!ok) {
|
||||
throw new Error(`button not found: ${text}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function clickTab(win, text) {
|
||||
const ok = await exec(win, `(() => {
|
||||
const target = Array.from(document.querySelectorAll('[role="tab"]')).find((el) => (el.textContent || '').includes(${JSON.stringify(text)}))
|
||||
if (!target) return false
|
||||
target.scrollIntoView({ block: 'center' })
|
||||
target.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true, cancelable: true, pointerId: 1, button: 0, pointerType: 'mouse', isPrimary: true }))
|
||||
target.dispatchEvent(new MouseEvent('mousedown', { bubbles: true, cancelable: true, button: 0 }))
|
||||
target.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, cancelable: true, button: 0 }))
|
||||
target.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, button: 0 }))
|
||||
return true
|
||||
})()`)
|
||||
if (!ok) {
|
||||
throw new Error(`tab not found: ${text}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function clickGraphElement(win, selector, index = 0) {
|
||||
const rect = await exec(win, `(() => {
|
||||
const targets = Array.from(document.querySelectorAll(${JSON.stringify(selector)}))
|
||||
const target = targets[${index}]
|
||||
if (!target) return null
|
||||
target.scrollIntoView({ block: 'center', inline: 'center' })
|
||||
const r = target.getBoundingClientRect()
|
||||
return { x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2) }
|
||||
})()`)
|
||||
if (!rect) {
|
||||
throw new Error(`graph element not found: ${selector}[${index}]`)
|
||||
}
|
||||
await sendClick(win, rect.x, rect.y)
|
||||
}
|
||||
|
||||
async function capturePluginFilterState(win) {
|
||||
await win.loadURL(`${DASHBOARD_URL}/plugin-config`)
|
||||
await waitFor(
|
||||
win,
|
||||
`document.body && document.body.innerText.includes('插件配置') && document.querySelector('input[placeholder="搜索插件..."]')`,
|
||||
'plugin config page',
|
||||
30000,
|
||||
400,
|
||||
)
|
||||
await exec(win, `(() => {
|
||||
const input = document.querySelector('input[placeholder="搜索插件..."]')
|
||||
if (!input) return false
|
||||
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set
|
||||
setter?.call(input, 'memorix')
|
||||
input.dispatchEvent(new Event('input', { bubbles: true }))
|
||||
input.dispatchEvent(new Event('change', { bubbles: true }))
|
||||
return true
|
||||
})()`)
|
||||
await wait(500)
|
||||
await capture(win, '01-plugin-config-filtered.png')
|
||||
}
|
||||
|
||||
app.whenReady().then(async () => {
|
||||
const win = new BrowserWindow({
|
||||
width: 1600,
|
||||
height: 1200,
|
||||
show: false,
|
||||
webPreferences: {
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
},
|
||||
})
|
||||
|
||||
await win.loadURL(`${DASHBOARD_URL}/auth`)
|
||||
await waitFor(win, `document.readyState === 'complete'`, 'auth page')
|
||||
await capture(win, '00-auth-login.png')
|
||||
await setSessionCookie(win)
|
||||
|
||||
await capturePluginFilterState(win)
|
||||
|
||||
await win.loadURL(`${DASHBOARD_URL}/resource/knowledge-base`)
|
||||
await waitFor(
|
||||
win,
|
||||
`document.body && document.body.innerText.includes('运行时自检') && document.body.innerText.includes('刷新数据')`,
|
||||
'memory console ready',
|
||||
30000,
|
||||
500,
|
||||
)
|
||||
await capture(win, '02-memory-console-before-import.png')
|
||||
|
||||
const beforeGraph = await getJson(win, '/api/webui/memory/graph?limit=120')
|
||||
const beforeTasks = await getJson(win, '/api/webui/memory/import/tasks?limit=20')
|
||||
const knownTaskIds = new Set(
|
||||
Array.isArray(beforeTasks.items)
|
||||
? beforeTasks.items.map((item) => String(item.task_id || item.taskId || ''))
|
||||
: [],
|
||||
)
|
||||
|
||||
await openImportTab(win)
|
||||
await setJsonMode(win)
|
||||
await typeIntoLabeled(win, '名称', 'input', sampleName)
|
||||
await typeIntoLabeled(win, '粘贴内容', 'textarea', sampleJson)
|
||||
await capture(win, '03-memory-import-json-filled.png')
|
||||
|
||||
await clickButton(win, '创建导入任务')
|
||||
|
||||
let taskId = null
|
||||
let taskStatus = null
|
||||
const start = Date.now()
|
||||
while (Date.now() - start < 120000) {
|
||||
const payload = await getJson(win, '/api/webui/memory/import/tasks?limit=20')
|
||||
fs.writeFileSync(path.join(OUTPUT_DIR, 'tasks-last.json'), JSON.stringify(payload, null, 2))
|
||||
const items = Array.isArray(payload.items) ? payload.items : []
|
||||
const task = items.find((item) => !knownTaskIds.has(String(item.task_id || item.taskId || '')))
|
||||
if (task) {
|
||||
taskId = task.task_id || task.taskId || null
|
||||
taskStatus = task.status || null
|
||||
if (['completed', 'failed', 'cancelled'].includes(String(taskStatus))) {
|
||||
break
|
||||
}
|
||||
}
|
||||
await wait(1500)
|
||||
}
|
||||
|
||||
if (!taskId) {
|
||||
throw new Error('new json import task not observed')
|
||||
}
|
||||
|
||||
const detail = await getJson(
|
||||
win,
|
||||
`/api/webui/memory/import/tasks/${encodeURIComponent(taskId)}?include_chunks=true`,
|
||||
)
|
||||
fs.writeFileSync(path.join(OUTPUT_DIR, 'task-detail.json'), JSON.stringify(detail, null, 2))
|
||||
fs.writeFileSync(
|
||||
path.join(OUTPUT_DIR, 'task-status.txt'),
|
||||
`taskId=${taskId}\nstatus=${taskStatus}\nsource=${sampleSource}\n`,
|
||||
)
|
||||
|
||||
await clickButton(win, '刷新数据')
|
||||
await wait(2000)
|
||||
await capture(win, '04-memory-console-after-import.png')
|
||||
|
||||
await win.loadURL(`${DASHBOARD_URL}/resource/knowledge-graph`)
|
||||
await waitFor(
|
||||
win,
|
||||
`document.body && document.body.innerText.includes('长期记忆图谱') && document.body.innerText.includes('实体关系图') && document.body.innerText.includes('证据视图')`,
|
||||
'graph page ready',
|
||||
30000,
|
||||
400,
|
||||
)
|
||||
await wait(3000)
|
||||
const afterGraph = await getJson(win, '/api/webui/memory/graph?limit=120')
|
||||
fs.writeFileSync(path.join(OUTPUT_DIR, 'graph-after.json'), JSON.stringify(afterGraph, null, 2))
|
||||
await capture(win, '05-memory-graph-after-import.png')
|
||||
|
||||
if (Array.isArray(afterGraph.nodes) && afterGraph.nodes.length > 0) {
|
||||
await clickGraphElement(win, '.react-flow__node', 0)
|
||||
await waitFor(win, `document.body && document.body.innerText.includes('实体详情')`, 'node detail dialog', 10000, 250)
|
||||
await capture(win, '06-memory-node-detail.png')
|
||||
try {
|
||||
await clickButton(win, '切到证据视图')
|
||||
await waitFor(
|
||||
win,
|
||||
`document.body && document.body.innerText.includes('证据视图') && document.querySelectorAll('.react-flow__node').length > 0`,
|
||||
'evidence graph after node click',
|
||||
10000,
|
||||
250,
|
||||
)
|
||||
await capture(win, '07-memory-evidence-view.png')
|
||||
} catch (error) {
|
||||
fs.writeFileSync(path.join(OUTPUT_DIR, '07-memory-evidence-view-error.txt'), String(error?.stack || error))
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(afterGraph.edges) && afterGraph.edges.length > 0) {
|
||||
try {
|
||||
await clickTab(win, '实体关系图')
|
||||
await wait(800)
|
||||
await clickGraphElement(win, '.react-flow__edge', 0)
|
||||
await waitFor(win, `document.body && document.body.innerText.includes('关系详情')`, 'edge detail dialog', 10000, 250)
|
||||
await capture(win, '08-memory-edge-detail.png')
|
||||
} catch (error) {
|
||||
fs.writeFileSync(path.join(OUTPUT_DIR, '08-memory-edge-detail-error.txt'), String(error?.stack || error))
|
||||
}
|
||||
}
|
||||
|
||||
const summary = {
|
||||
before: {
|
||||
nodes: beforeGraph.total_nodes,
|
||||
edges: beforeGraph.total_edges,
|
||||
},
|
||||
after: {
|
||||
nodes: afterGraph.total_nodes,
|
||||
edges: afterGraph.total_edges,
|
||||
},
|
||||
taskId,
|
||||
taskStatus,
|
||||
source: sampleSource,
|
||||
inputMode: detail?.task?.files?.[0]?.input_mode || null,
|
||||
strategyType: detail?.task?.files?.[0]?.detected_strategy_type || null,
|
||||
fileStatus: detail?.task?.files?.[0]?.status || null,
|
||||
outputDir: OUTPUT_DIR,
|
||||
}
|
||||
fs.writeFileSync(path.join(OUTPUT_DIR, 'validation-summary.json'), JSON.stringify(summary, null, 2))
|
||||
console.log(JSON.stringify(summary, null, 2))
|
||||
|
||||
await win.close()
|
||||
app.quit()
|
||||
}).catch((error) => {
|
||||
console.error(error)
|
||||
app.exit(1)
|
||||
})
|
||||
@@ -2,6 +2,7 @@ import { useEffect, useState } from 'react'
|
||||
import CodeMirror from '@uiw/react-codemirror'
|
||||
import { css } from '@codemirror/lang-css'
|
||||
import { json, jsonParseLinter } from '@codemirror/lang-json'
|
||||
import { linter } from '@codemirror/lint'
|
||||
import { python } from '@codemirror/lang-python'
|
||||
import { oneDark } from '@codemirror/theme-one-dark'
|
||||
import { EditorView } from '@codemirror/view'
|
||||
@@ -29,7 +30,7 @@ interface CodeEditorProps {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const languageExtensions: Record<Language, any[]> = {
|
||||
python: [python()],
|
||||
json: [json(), jsonParseLinter()],
|
||||
json: [json(), linter(jsonParseLinter())],
|
||||
toml: [StreamLanguage.define(tomlMode)],
|
||||
css: [css()],
|
||||
text: [],
|
||||
|
||||
311
dashboard/src/components/memory/MemoryConfigEditor.tsx
Normal file
311
dashboard/src/components/memory/MemoryConfigEditor.tsx
Normal file
@@ -0,0 +1,311 @@
|
||||
import { useMemo, useState } from 'react'
|
||||
|
||||
import { ListFieldEditor } from '@/components'
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Label } from '@/components/ui/label'
|
||||
import { Switch } from '@/components/ui/switch'
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'
|
||||
import { Textarea } from '@/components/ui/textarea'
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '@/components/ui/select'
|
||||
import type { ConfigFieldSchema, PluginConfigSchema } from '@/lib/plugin-api'
|
||||
|
||||
interface MemoryConfigEditorProps {
|
||||
schema: PluginConfigSchema
|
||||
config: Record<string, unknown>
|
||||
onChange: (nextConfig: Record<string, unknown>) => void
|
||||
disabled?: boolean
|
||||
}
|
||||
|
||||
function getNestedRecord(config: Record<string, unknown>, path: string): Record<string, unknown> | undefined {
|
||||
const parts = path.split('.').filter(Boolean)
|
||||
let current: unknown = config
|
||||
|
||||
for (const part of parts) {
|
||||
if (!current || typeof current !== 'object' || Array.isArray(current)) {
|
||||
return undefined
|
||||
}
|
||||
current = (current as Record<string, unknown>)[part]
|
||||
}
|
||||
|
||||
if (!current || typeof current !== 'object' || Array.isArray(current)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return current as Record<string, unknown>
|
||||
}
|
||||
|
||||
function setNestedField(
|
||||
config: Record<string, unknown>,
|
||||
path: string,
|
||||
fieldName: string,
|
||||
value: unknown,
|
||||
): Record<string, unknown> {
|
||||
const parts = path.split('.').filter(Boolean)
|
||||
const nextConfig: Record<string, unknown> = { ...config }
|
||||
let target = nextConfig
|
||||
let source: Record<string, unknown> | undefined = config
|
||||
|
||||
for (const part of parts) {
|
||||
const sourceValue: unknown = source?.[part]
|
||||
const nextValue =
|
||||
sourceValue && typeof sourceValue === 'object' && !Array.isArray(sourceValue)
|
||||
? { ...(sourceValue as Record<string, unknown>) }
|
||||
: {}
|
||||
target[part] = nextValue
|
||||
target = nextValue
|
||||
source =
|
||||
sourceValue && typeof sourceValue === 'object' && !Array.isArray(sourceValue)
|
||||
? (sourceValue as Record<string, unknown>)
|
||||
: undefined
|
||||
}
|
||||
|
||||
target[fieldName] = value
|
||||
return nextConfig
|
||||
}
|
||||
|
||||
function FieldRenderer({
|
||||
field,
|
||||
value,
|
||||
onChange,
|
||||
disabled,
|
||||
}: {
|
||||
field: ConfigFieldSchema
|
||||
value: unknown
|
||||
onChange: (value: unknown) => void
|
||||
disabled?: boolean
|
||||
}) {
|
||||
const [jsonDraft, setJsonDraft] = useState(
|
||||
typeof value === 'string' ? String(value) : JSON.stringify(value ?? field.default ?? {}, null, 2),
|
||||
)
|
||||
|
||||
switch (field.ui_type) {
|
||||
case 'switch':
|
||||
return (
|
||||
<div className="flex items-center justify-between rounded-lg border bg-background px-4 py-3">
|
||||
<div className="space-y-1">
|
||||
<Label>{field.label}</Label>
|
||||
{field.hint && <p className="text-xs text-muted-foreground">{field.hint}</p>}
|
||||
</div>
|
||||
<Switch
|
||||
checked={Boolean(value ?? field.default)}
|
||||
onCheckedChange={onChange}
|
||||
disabled={disabled || field.disabled}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
|
||||
case 'number':
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<Label>{field.label}</Label>
|
||||
<Input
|
||||
type="number"
|
||||
value={String(value ?? field.default ?? '')}
|
||||
onChange={(event) => onChange(Number(event.target.value))}
|
||||
min={field.min}
|
||||
max={field.max}
|
||||
step={field.step ?? 1}
|
||||
disabled={disabled || field.disabled}
|
||||
placeholder={field.placeholder}
|
||||
/>
|
||||
{field.hint && <p className="text-xs text-muted-foreground">{field.hint}</p>}
|
||||
</div>
|
||||
)
|
||||
|
||||
case 'select':
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<Label>{field.label}</Label>
|
||||
<Select
|
||||
value={String(value ?? field.default ?? '')}
|
||||
onValueChange={onChange}
|
||||
disabled={disabled || field.disabled}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder={field.placeholder ?? '请选择'} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{(field.choices ?? []).map((choice) => (
|
||||
<SelectItem key={String(choice)} value={String(choice)}>
|
||||
{String(choice)}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{field.hint && <p className="text-xs text-muted-foreground">{field.hint}</p>}
|
||||
</div>
|
||||
)
|
||||
|
||||
case 'textarea':
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<Label>{field.label}</Label>
|
||||
<Textarea
|
||||
value={String(value ?? field.default ?? '')}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
rows={field.rows ?? 4}
|
||||
placeholder={field.placeholder}
|
||||
disabled={disabled || field.disabled}
|
||||
/>
|
||||
{field.hint && <p className="text-xs text-muted-foreground">{field.hint}</p>}
|
||||
</div>
|
||||
)
|
||||
|
||||
case 'list':
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<Label>{field.label}</Label>
|
||||
<ListFieldEditor
|
||||
value={Array.isArray(value) ? value : (Array.isArray(field.default) ? field.default : [])}
|
||||
onChange={onChange as (value: unknown[]) => void}
|
||||
itemType={field.item_type}
|
||||
itemFields={field.item_fields}
|
||||
minItems={field.min_items}
|
||||
maxItems={field.max_items}
|
||||
placeholder={field.placeholder}
|
||||
disabled={disabled || field.disabled}
|
||||
/>
|
||||
{field.hint && <p className="text-xs text-muted-foreground">{field.hint}</p>}
|
||||
</div>
|
||||
)
|
||||
|
||||
case 'json':
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<Label>{field.label}</Label>
|
||||
<Textarea
|
||||
value={jsonDraft}
|
||||
rows={field.rows ?? 6}
|
||||
disabled={disabled || field.disabled}
|
||||
onChange={(event) => {
|
||||
const nextValue = event.target.value
|
||||
setJsonDraft(nextValue)
|
||||
try {
|
||||
onChange(JSON.parse(nextValue))
|
||||
} catch {
|
||||
// keep draft until valid JSON
|
||||
}
|
||||
}}
|
||||
/>
|
||||
{field.hint && <p className="text-xs text-muted-foreground">{field.hint}</p>}
|
||||
</div>
|
||||
)
|
||||
|
||||
default:
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<Label>{field.label}</Label>
|
||||
<Input
|
||||
value={String(value ?? field.default ?? '')}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
disabled={disabled || field.disabled}
|
||||
placeholder={field.placeholder}
|
||||
/>
|
||||
{field.hint && <p className="text-xs text-muted-foreground">{field.hint}</p>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function SectionCard({
|
||||
sectionName,
|
||||
schema,
|
||||
config,
|
||||
onChange,
|
||||
disabled,
|
||||
}: {
|
||||
sectionName: string
|
||||
schema: PluginConfigSchema
|
||||
config: Record<string, unknown>
|
||||
onChange: (nextConfig: Record<string, unknown>) => void
|
||||
disabled?: boolean
|
||||
}) {
|
||||
const section = schema.sections[sectionName]
|
||||
if (!section) {
|
||||
return null
|
||||
}
|
||||
|
||||
const sectionValues = getNestedRecord(config, sectionName) ?? {}
|
||||
const orderedFields = Object.values(section.fields).sort((left, right) => left.order - right.order)
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>{section.title}</CardTitle>
|
||||
{section.description && <CardDescription>{section.description}</CardDescription>}
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
{orderedFields.map((field) => (
|
||||
<FieldRenderer
|
||||
key={`${sectionName}.${field.name}`}
|
||||
field={field}
|
||||
value={sectionValues[field.name]}
|
||||
disabled={disabled}
|
||||
onChange={(value) => onChange(setNestedField(config, sectionName, field.name, value))}
|
||||
/>
|
||||
))}
|
||||
</CardContent>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
export function MemoryConfigEditor({ schema, config, onChange, disabled }: MemoryConfigEditorProps) {
|
||||
const tabs = useMemo(
|
||||
() => [...(schema.layout.tabs ?? [])].sort((left, right) => left.order - right.order),
|
||||
[schema.layout.tabs],
|
||||
)
|
||||
|
||||
if (tabs.length === 0) {
|
||||
const orderedSections = Object.keys(schema.sections).sort(
|
||||
(left, right) => (schema.sections[left]?.order ?? 0) - (schema.sections[right]?.order ?? 0),
|
||||
)
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
{orderedSections.map((sectionName) => (
|
||||
<SectionCard
|
||||
key={sectionName}
|
||||
sectionName={sectionName}
|
||||
schema={schema}
|
||||
config={config}
|
||||
onChange={onChange}
|
||||
disabled={disabled}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<Tabs defaultValue={tabs[0]?.id} className="space-y-4">
|
||||
<TabsList className="h-auto flex-wrap justify-start">
|
||||
{tabs.map((tab) => (
|
||||
<TabsTrigger key={tab.id} value={tab.id}>
|
||||
{tab.title}
|
||||
</TabsTrigger>
|
||||
))}
|
||||
</TabsList>
|
||||
|
||||
{tabs.map((tab) => (
|
||||
<TabsContent key={tab.id} value={tab.id} className="space-y-4">
|
||||
{tab.sections.map((sectionName) => (
|
||||
<SectionCard
|
||||
key={sectionName}
|
||||
sectionName={sectionName}
|
||||
schema={schema}
|
||||
config={config}
|
||||
onChange={onChange}
|
||||
disabled={disabled}
|
||||
/>
|
||||
))}
|
||||
</TabsContent>
|
||||
))}
|
||||
</Tabs>
|
||||
)
|
||||
}
|
||||
281
dashboard/src/components/memory/MemoryDeleteDialog.tsx
Normal file
281
dashboard/src/components/memory/MemoryDeleteDialog.tsx
Normal file
@@ -0,0 +1,281 @@
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import { AlertTriangle, RotateCcw, Search, Trash2 } from 'lucide-react'
|
||||
|
||||
import { Alert, AlertDescription } from '@/components/ui/alert'
|
||||
import { Badge } from '@/components/ui/badge'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import {
|
||||
Dialog,
|
||||
DialogBody,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { ScrollArea } from '@/components/ui/scroll-area'
|
||||
import type {
|
||||
MemoryDeleteExecutePayload,
|
||||
MemoryDeletePreviewItemPayload,
|
||||
MemoryDeletePreviewPayload,
|
||||
} from '@/lib/memory-api'
|
||||
|
||||
const DELETE_PREVIEW_PAGE_SIZE = 8
|
||||
|
||||
function formatMode(mode: string): string {
|
||||
switch (mode) {
|
||||
case 'entity':
|
||||
return '实体删除'
|
||||
case 'relation':
|
||||
return '关系删除'
|
||||
case 'paragraph':
|
||||
return '段落删除'
|
||||
case 'source':
|
||||
return '来源删除'
|
||||
case 'mixed':
|
||||
return '混合删除'
|
||||
default:
|
||||
return mode || '删除'
|
||||
}
|
||||
}
|
||||
|
||||
function formatCountLabel(label: string, value: number): string {
|
||||
return `${label} ${value}`
|
||||
}
|
||||
|
||||
function PreviewItemList({ items }: { items: MemoryDeletePreviewItemPayload[] }) {
|
||||
if (items.length <= 0) {
|
||||
return <p className="text-sm text-muted-foreground">当前预览没有可展示的明细项。</p>
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
{items.slice(0, 16).map((item) => (
|
||||
<div key={`${item.item_type}:${item.item_hash}:${item.item_key ?? ''}`} className="rounded-lg border bg-muted/30 p-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge variant="outline">{item.item_type}</Badge>
|
||||
{item.source ? <Badge variant="secondary">{item.source}</Badge> : null}
|
||||
</div>
|
||||
<div className="mt-2 text-sm font-medium break-words">{item.label || item.item_key || item.item_hash}</div>
|
||||
{item.preview ? <div className="mt-1 text-xs text-muted-foreground break-words">{item.preview}</div> : null}
|
||||
<code className="mt-2 block break-all text-[11px] text-muted-foreground">{item.item_hash || item.item_key}</code>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
interface MemoryDeleteDialogProps {
|
||||
open: boolean
|
||||
onOpenChange: (open: boolean) => void
|
||||
title: string
|
||||
description?: string
|
||||
preview: MemoryDeletePreviewPayload | null
|
||||
result: MemoryDeleteExecutePayload | null
|
||||
loadingPreview?: boolean
|
||||
executing?: boolean
|
||||
restoring?: boolean
|
||||
error?: string | null
|
||||
onExecute: () => void
|
||||
onRestore?: () => void
|
||||
}
|
||||
|
||||
export function MemoryDeleteDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
title,
|
||||
description,
|
||||
preview,
|
||||
result,
|
||||
loadingPreview = false,
|
||||
executing = false,
|
||||
restoring = false,
|
||||
error,
|
||||
onExecute,
|
||||
onRestore,
|
||||
}: MemoryDeleteDialogProps) {
|
||||
const [itemSearch, setItemSearch] = useState('')
|
||||
const [itemPage, setItemPage] = useState(1)
|
||||
const counts = preview?.counts ?? result?.counts ?? {}
|
||||
const previewSources = Array.isArray(preview?.sources) ? preview.sources : []
|
||||
const previewItems = Array.isArray(preview?.items) ? preview.items : []
|
||||
const filteredPreviewItems = useMemo(() => {
|
||||
const keyword = itemSearch.trim().toLowerCase()
|
||||
if (!keyword) {
|
||||
return previewItems
|
||||
}
|
||||
return previewItems.filter((item) =>
|
||||
[
|
||||
item.item_type,
|
||||
item.item_hash,
|
||||
item.item_key,
|
||||
item.label,
|
||||
item.preview,
|
||||
item.source,
|
||||
]
|
||||
.map((value) => String(value ?? '').toLowerCase())
|
||||
.some((value) => value.includes(keyword)),
|
||||
)
|
||||
}, [itemSearch, previewItems])
|
||||
const itemPageCount = Math.max(1, Math.ceil(filteredPreviewItems.length / DELETE_PREVIEW_PAGE_SIZE))
|
||||
const pagedPreviewItems = useMemo(() => {
|
||||
const start = (itemPage - 1) * DELETE_PREVIEW_PAGE_SIZE
|
||||
return filteredPreviewItems.slice(start, start + DELETE_PREVIEW_PAGE_SIZE)
|
||||
}, [filteredPreviewItems, itemPage])
|
||||
const countBadges = [
|
||||
{ key: 'entities', label: '实体', value: Number(counts.entities ?? 0) },
|
||||
{ key: 'relations', label: '关系', value: Number(counts.relations ?? 0) },
|
||||
{ key: 'paragraphs', label: '段落', value: Number(counts.paragraphs ?? 0) },
|
||||
{ key: 'sources', label: '来源', value: Number(counts.sources ?? 0) },
|
||||
].filter((item) => item.value > 0)
|
||||
|
||||
useEffect(() => {
|
||||
setItemSearch('')
|
||||
setItemPage(1)
|
||||
}, [preview?.mode, preview?.item_count, open])
|
||||
|
||||
useEffect(() => {
|
||||
setItemPage(1)
|
||||
}, [itemSearch])
|
||||
|
||||
useEffect(() => {
|
||||
if (itemPage > itemPageCount) {
|
||||
setItemPage(itemPageCount)
|
||||
}
|
||||
}, [itemPage, itemPageCount])
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="max-w-3xl max-h-[85vh] grid grid-rows-[auto_1fr_auto]" confirmOnEnter>
|
||||
<DialogHeader>
|
||||
<DialogTitle className="flex items-center gap-2">
|
||||
<AlertTriangle className="h-5 w-5 text-amber-500" />
|
||||
{title}
|
||||
</DialogTitle>
|
||||
{description ? <DialogDescription>{description}</DialogDescription> : null}
|
||||
</DialogHeader>
|
||||
|
||||
<DialogBody className="space-y-4 overflow-y-auto">
|
||||
{loadingPreview ? (
|
||||
<div className="rounded-lg border bg-muted/30 p-4 text-sm text-muted-foreground">正在生成删除预览...</div>
|
||||
) : null}
|
||||
|
||||
{error ? (
|
||||
<Alert variant="destructive">
|
||||
<AlertDescription>{error}</AlertDescription>
|
||||
</Alert>
|
||||
) : null}
|
||||
|
||||
{preview ? (
|
||||
<>
|
||||
<div className="rounded-xl border bg-muted/30 p-4">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge>{formatMode(preview.mode)}</Badge>
|
||||
<Badge variant="secondary">{formatCountLabel('预览项', Number(preview.item_count ?? previewItems.length))}</Badge>
|
||||
{countBadges.map((item) => (
|
||||
<Badge key={item.key} variant="outline">
|
||||
{formatCountLabel(item.label, item.value)}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
{previewSources.length > 0 ? (
|
||||
<div className="mt-3 text-sm text-muted-foreground break-words">
|
||||
关联来源：{previewSources.join('、')}
|
||||
</div>
|
||||
) : null}
|
||||
{preview.matched_source_count ? (
|
||||
<div className="mt-2 text-xs text-muted-foreground">
|
||||
命中来源 {preview.matched_source_count}
|
||||
{preview.requested_source_count ? ` / 请求来源 ${preview.requested_source_count}` : ''}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<div className="flex flex-col gap-3 md:flex-row md:items-center md:justify-between">
|
||||
<div>
|
||||
<div className="text-sm font-semibold">本次将删除的对象摘要</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
命中 {filteredPreviewItems.length} / {previewItems.length} 项
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex flex-col gap-2 md:min-w-[300px]">
|
||||
<div className="relative">
|
||||
<Search className="pointer-events-none absolute left-3 top-1/2 h-3.5 w-3.5 -translate-y-1/2 text-muted-foreground" />
|
||||
<Input
|
||||
value={itemSearch}
|
||||
onChange={(event) => setItemSearch(event.target.value)}
|
||||
placeholder="搜索类型 / hash / item_key / source"
|
||||
className="pl-8"
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center justify-between text-xs text-muted-foreground">
|
||||
<span>第 {itemPage} / {itemPageCount} 页</span>
|
||||
<span>每页 {DELETE_PREVIEW_PAGE_SIZE} 项</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<ScrollArea className="h-[320px] rounded-lg border bg-background/60">
|
||||
<div className="p-3">
|
||||
<PreviewItemList items={pagedPreviewItems} />
|
||||
</div>
|
||||
</ScrollArea>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => setItemPage((current) => Math.max(1, current - 1))}
|
||||
disabled={itemPage <= 1}
|
||||
>
|
||||
上一页
|
||||
</Button>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
支持按对象类型、hash、item_key、source 和预览内容检索
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => setItemPage((current) => Math.min(itemPageCount, current + 1))}
|
||||
disabled={itemPage >= itemPageCount}
|
||||
>
|
||||
下一页
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
) : null}
|
||||
|
||||
{result?.success ? (
|
||||
<Alert>
|
||||
<AlertDescription className="space-y-1">
|
||||
<div>删除执行成功，操作 ID：<code>{result.operation_id}</code></div>
|
||||
<div>
|
||||
实际删除：实体 {result.deleted_entity_count}，关系 {result.deleted_relation_count}，段落 {result.deleted_paragraph_count}，来源 {result.deleted_source_count}
|
||||
</div>
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
) : null}
|
||||
</DialogBody>
|
||||
|
||||
<DialogFooter>
|
||||
<Button variant="outline" onClick={() => onOpenChange(false)}>
|
||||
关闭
|
||||
</Button>
|
||||
{result?.success && onRestore ? (
|
||||
<Button variant="outline" onClick={onRestore} disabled={restoring}>
|
||||
<RotateCcw className="mr-2 h-4 w-4" />
|
||||
{restoring ? '恢复中...' : '恢复本次删除'}
|
||||
</Button>
|
||||
) : null}
|
||||
{!result?.success ? (
|
||||
<Button data-dialog-action="confirm" variant="destructive" onClick={onExecute} disabled={loadingPreview || executing || !preview}>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
{executing ? '执行中...' : '确认删除'}
|
||||
</Button>
|
||||
) : null}
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
@@ -32,8 +32,8 @@
|
||||
"expressionManagement": "Expression Management",
|
||||
"slangManagement": "Slang Management",
|
||||
"personInfo": "Person Info",
|
||||
"knowledgeGraph": "Knowledge Graph",
|
||||
"knowledgeBase": "Knowledge Base",
|
||||
"knowledgeGraph": "Long-Term Memory Graph",
|
||||
"knowledgeBase": "Long-Term Memory Console",
|
||||
"pluginMarket": "Plugin Market",
|
||||
"configTemplate": "Config Templates",
|
||||
"pluginConfig": "Plugin Config",
|
||||
|
||||
@@ -32,8 +32,8 @@
|
||||
"expressionManagement": "表現管理",
|
||||
"slangManagement": "スラング管理",
|
||||
"personInfo": "人物情報",
|
||||
"knowledgeGraph": "知識グラフ",
|
||||
"knowledgeBase": "ナレッジベース",
|
||||
"knowledgeGraph": "長期記憶グラフ",
|
||||
"knowledgeBase": "長期記憶コンソール",
|
||||
"pluginMarket": "プラグインマーケット",
|
||||
"configTemplate": "設定テンプレート",
|
||||
"pluginConfig": "プラグイン設定",
|
||||
|
||||
@@ -32,8 +32,8 @@
|
||||
"expressionManagement": "표현 관리",
|
||||
"slangManagement": "슬랭 관리",
|
||||
"personInfo": "인물 정보",
|
||||
"knowledgeGraph": "지식 그래프",
|
||||
"knowledgeBase": "지식 베이스",
|
||||
"knowledgeGraph": "장기 기억 그래프",
|
||||
"knowledgeBase": "장기 기억 콘솔",
|
||||
"pluginMarket": "플러그인 마켓",
|
||||
"configTemplate": "설정 템플릿",
|
||||
"pluginConfig": "플러그인 설정",
|
||||
|
||||
@@ -32,8 +32,8 @@
|
||||
"expressionManagement": "表达方式管理",
|
||||
"slangManagement": "黑话管理",
|
||||
"personInfo": "人物信息管理",
|
||||
"knowledgeGraph": "知识库图谱可视化",
|
||||
"knowledgeBase": "麦麦知识库管理",
|
||||
"knowledgeGraph": "长期记忆图谱",
|
||||
"knowledgeBase": "长期记忆控制台",
|
||||
"pluginMarket": "插件市场",
|
||||
"configTemplate": "配置模板市场",
|
||||
"pluginConfig": "插件配置",
|
||||
|
||||
509
dashboard/src/lib/memory-api.ts
Normal file
509
dashboard/src/lib/memory-api.ts
Normal file
@@ -0,0 +1,509 @@
|
||||
import type { PluginConfigSchema } from '@/lib/plugin-api'
|
||||
|
||||
import { getApiBaseUrl } from './api-base'
|
||||
import { isElectron } from './runtime'
|
||||
|
||||
async function getMemoryApiBase(): Promise<string> {
|
||||
if (isElectron()) {
|
||||
const base = await getApiBaseUrl()
|
||||
return base ? `${base}/api/webui/memory` : '/api/webui/memory'
|
||||
}
|
||||
return import.meta.env.VITE_API_BASE_URL
|
||||
? `${import.meta.env.VITE_API_BASE_URL}/memory`
|
||||
: '/api/webui/memory'
|
||||
}
|
||||
|
||||
async function requestJson<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
const response = await fetch(`${await getMemoryApiBase()}${path}`, init)
|
||||
if (!response.ok) {
|
||||
let detail = `${response.status}`
|
||||
try {
|
||||
const payload = await response.json()
|
||||
detail = String(payload?.detail ?? payload?.error ?? detail)
|
||||
} catch {
|
||||
// ignore json parsing fallback
|
||||
}
|
||||
throw new Error(detail)
|
||||
}
|
||||
return response.json() as Promise<T>
|
||||
}
|
||||
|
||||
export interface MemoryGraphNodePayload {
|
||||
id: string
|
||||
name: string
|
||||
attributes?: Record<string, unknown>
|
||||
}
|
||||
|
||||
export interface MemoryGraphEdgePayload {
|
||||
source: string
|
||||
target: string
|
||||
weight: number
|
||||
relation_hashes?: string[]
|
||||
predicates?: string[]
|
||||
relation_count?: number
|
||||
evidence_count?: number
|
||||
label?: string
|
||||
}
|
||||
|
||||
export interface MemoryGraphPayload {
|
||||
success: boolean
|
||||
nodes: MemoryGraphNodePayload[]
|
||||
edges: MemoryGraphEdgePayload[]
|
||||
total_nodes: number
|
||||
total_edges: number
|
||||
}
|
||||
|
||||
export interface MemoryGraphRelationDetailPayload {
|
||||
hash: string
|
||||
subject: string
|
||||
predicate: string
|
||||
object: string
|
||||
text: string
|
||||
confidence: number
|
||||
paragraph_count: number
|
||||
paragraph_hashes: string[]
|
||||
source_paragraph: string
|
||||
}
|
||||
|
||||
export interface MemoryGraphParagraphDetailPayload {
|
||||
hash: string
|
||||
content: string
|
||||
preview: string
|
||||
source: string
|
||||
created_at?: number | null
|
||||
updated_at?: number | null
|
||||
entity_count: number
|
||||
relation_count: number
|
||||
entities: string[]
|
||||
relations: string[]
|
||||
}
|
||||
|
||||
export interface MemoryEvidenceGraphNodePayload {
|
||||
id: string
|
||||
type: 'entity' | 'relation' | 'paragraph'
|
||||
content: string
|
||||
metadata?: MemoryEvidenceGraphNodeMetadata
|
||||
}
|
||||
|
||||
export interface MemoryEvidenceGraphEdgePayload {
|
||||
source: string
|
||||
target: string
|
||||
kind: 'mentions' | 'supports' | 'subject' | 'object'
|
||||
label: string
|
||||
weight: number
|
||||
}
|
||||
|
||||
export interface MemoryEvidenceGraphPayload {
|
||||
nodes: MemoryEvidenceGraphNodePayload[]
|
||||
edges: MemoryEvidenceGraphEdgePayload[]
|
||||
focus_entities: string[]
|
||||
}
|
||||
|
||||
export interface MemoryEvidenceEntityNodeMetadata extends Record<string, unknown> {
|
||||
entity_name?: string
|
||||
}
|
||||
|
||||
export interface MemoryEvidenceRelationNodeMetadata extends Record<string, unknown> {
|
||||
hash?: string
|
||||
subject?: string
|
||||
predicate?: string
|
||||
object?: string
|
||||
confidence?: number
|
||||
paragraph_count?: number
|
||||
paragraph_hashes?: string[]
|
||||
text?: string
|
||||
}
|
||||
|
||||
export interface MemoryEvidenceParagraphNodeMetadata extends Record<string, unknown> {
|
||||
hash?: string
|
||||
source?: string
|
||||
updated_at?: number | null
|
||||
entity_count?: number
|
||||
relation_count?: number
|
||||
preview?: string
|
||||
}
|
||||
|
||||
export type MemoryEvidenceGraphNodeMetadata =
|
||||
| MemoryEvidenceEntityNodeMetadata
|
||||
| MemoryEvidenceRelationNodeMetadata
|
||||
| MemoryEvidenceParagraphNodeMetadata
|
||||
| Record<string, unknown>
|
||||
|
||||
export interface MemoryGraphNodeDetailPayload {
|
||||
success: boolean
|
||||
node: {
|
||||
id: string
|
||||
type: 'entity'
|
||||
content: string
|
||||
hash?: string
|
||||
appearance_count?: number
|
||||
}
|
||||
relations: MemoryGraphRelationDetailPayload[]
|
||||
paragraphs: MemoryGraphParagraphDetailPayload[]
|
||||
evidence_graph: MemoryEvidenceGraphPayload
|
||||
}
|
||||
|
||||
export interface MemoryGraphEdgeDetailPayload {
|
||||
success: boolean
|
||||
edge: MemoryGraphEdgePayload
|
||||
relations: MemoryGraphRelationDetailPayload[]
|
||||
paragraphs: MemoryGraphParagraphDetailPayload[]
|
||||
evidence_graph: MemoryEvidenceGraphPayload
|
||||
}
|
||||
|
||||
export interface MemoryRuntimeConfigPayload {
|
||||
success: boolean
|
||||
config: Record<string, unknown>
|
||||
data_dir: string
|
||||
embedding_dimension: number
|
||||
auto_save: boolean
|
||||
relation_vectors_enabled: boolean
|
||||
runtime_ready: boolean
|
||||
embedding_degraded: boolean
|
||||
embedding_degraded_reason: string
|
||||
embedding_degraded_since?: number | null
|
||||
embedding_last_check?: number | null
|
||||
paragraph_vector_backfill_pending: number
|
||||
paragraph_vector_backfill_running: number
|
||||
paragraph_vector_backfill_failed: number
|
||||
paragraph_vector_backfill_done: number
|
||||
}
|
||||
|
||||
export interface MemoryRuntimeSelfCheckPayload {
|
||||
success: boolean
|
||||
report?: Record<string, unknown>
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface MemoryConfigPayload {
|
||||
success: boolean
|
||||
config: Record<string, unknown>
|
||||
path: string
|
||||
}
|
||||
|
||||
export interface MemoryRawConfigPayload {
|
||||
success: boolean
|
||||
config: string
|
||||
path: string
|
||||
}
|
||||
|
||||
export interface MemoryConfigSchemaPayload {
|
||||
success: boolean
|
||||
schema: PluginConfigSchema
|
||||
path: string
|
||||
}
|
||||
|
||||
export interface MemoryImportGuidePayload {
|
||||
success: boolean
|
||||
content: string
|
||||
source?: string
|
||||
path?: string
|
||||
settings?: Record<string, unknown>
|
||||
}
|
||||
|
||||
export interface MemoryTaskPayload {
|
||||
task_id?: string
|
||||
status?: string
|
||||
mode?: string
|
||||
created_at?: number
|
||||
updated_at?: number
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
export interface MemoryTaskListPayload {
|
||||
success: boolean
|
||||
items: MemoryTaskPayload[]
|
||||
count?: number
|
||||
settings?: Record<string, unknown>
|
||||
}
|
||||
|
||||
export interface MemoryTuningProfilePayload {
|
||||
success: boolean
|
||||
profile?: Record<string, unknown>
|
||||
settings?: Record<string, unknown>
|
||||
toml?: string
|
||||
}
|
||||
|
||||
export interface MemoryDeleteCountsPayload {
|
||||
relations?: number
|
||||
paragraphs?: number
|
||||
entities?: number
|
||||
sources?: number
|
||||
requested_sources?: number
|
||||
matched_sources?: number
|
||||
[key: string]: number | undefined
|
||||
}
|
||||
|
||||
export interface MemoryDeletePreviewItemPayload {
|
||||
item_type: string
|
||||
item_hash: string
|
||||
item_key?: string
|
||||
label?: string
|
||||
preview?: string
|
||||
source?: string
|
||||
}
|
||||
|
||||
export interface MemoryDeleteRequestPayload {
|
||||
mode: string
|
||||
selector: Record<string, unknown> | string
|
||||
reason?: string
|
||||
requested_by?: string
|
||||
}
|
||||
|
||||
export interface MemoryDeletePreviewPayload {
|
||||
success: boolean
|
||||
mode: string
|
||||
selector: Record<string, unknown> | string
|
||||
counts: MemoryDeleteCountsPayload
|
||||
sources: string[]
|
||||
items: MemoryDeletePreviewItemPayload[]
|
||||
item_count: number
|
||||
dry_run?: boolean
|
||||
requested_source_count?: number
|
||||
matched_source_count?: number
|
||||
vector_ids?: string[]
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface MemoryDeleteExecutePayload {
|
||||
success: boolean
|
||||
mode: string
|
||||
operation_id: string
|
||||
counts: MemoryDeleteCountsPayload
|
||||
sources: string[]
|
||||
deleted_count: number
|
||||
deleted_entity_count: number
|
||||
deleted_relation_count: number
|
||||
deleted_paragraph_count: number
|
||||
deleted_source_count: number
|
||||
deleted_vector_count?: number
|
||||
requested_source_count?: number
|
||||
matched_source_count?: number
|
||||
error?: string
|
||||
deleted?: boolean | number
|
||||
}
|
||||
|
||||
export interface MemoryDeleteOperationItemPayload {
|
||||
item_type: string
|
||||
item_hash: string
|
||||
item_key?: string
|
||||
payload?: Record<string, unknown>
|
||||
created_at?: number
|
||||
}
|
||||
|
||||
export interface MemoryDeleteOperationPayload {
|
||||
operation_id: string
|
||||
mode: string
|
||||
selector?: Record<string, unknown> | string
|
||||
reason?: string | null
|
||||
requested_by?: string | null
|
||||
status?: string
|
||||
created_at?: number
|
||||
restored_at?: number | null
|
||||
summary?: Record<string, unknown>
|
||||
items?: MemoryDeleteOperationItemPayload[]
|
||||
}
|
||||
|
||||
export interface MemoryDeleteOperationListPayload {
|
||||
success: boolean
|
||||
items: MemoryDeleteOperationPayload[]
|
||||
count?: number
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface MemoryDeleteOperationDetailPayload {
|
||||
success: boolean
|
||||
operation?: MemoryDeleteOperationPayload | null
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface MemorySourceItemPayload {
|
||||
source: string
|
||||
paragraph_count?: number
|
||||
relation_count?: number
|
||||
episode_rebuild_blocked?: boolean
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
export interface MemorySourceListPayload {
|
||||
success: boolean
|
||||
items: MemorySourceItemPayload[]
|
||||
count: number
|
||||
}
|
||||
|
||||
export async function getMemoryGraph(limit: number = 120): Promise<MemoryGraphPayload> {
|
||||
return requestJson<MemoryGraphPayload>(`/graph?limit=${limit}`)
|
||||
}
|
||||
|
||||
export async function getMemoryGraphNodeDetail(
|
||||
nodeId: string,
|
||||
options?: {
|
||||
relationLimit?: number
|
||||
paragraphLimit?: number
|
||||
evidenceNodeLimit?: number
|
||||
},
|
||||
): Promise<MemoryGraphNodeDetailPayload> {
|
||||
const params = new URLSearchParams({
|
||||
node_id: nodeId,
|
||||
relation_limit: String(options?.relationLimit ?? 20),
|
||||
paragraph_limit: String(options?.paragraphLimit ?? 20),
|
||||
evidence_node_limit: String(options?.evidenceNodeLimit ?? 80),
|
||||
})
|
||||
return requestJson<MemoryGraphNodeDetailPayload>(`/graph/node-detail?${params.toString()}`)
|
||||
}
|
||||
|
||||
export async function getMemoryGraphEdgeDetail(
|
||||
source: string,
|
||||
target: string,
|
||||
options?: {
|
||||
paragraphLimit?: number
|
||||
evidenceNodeLimit?: number
|
||||
},
|
||||
): Promise<MemoryGraphEdgeDetailPayload> {
|
||||
const params = new URLSearchParams({
|
||||
source,
|
||||
target,
|
||||
paragraph_limit: String(options?.paragraphLimit ?? 20),
|
||||
evidence_node_limit: String(options?.evidenceNodeLimit ?? 80),
|
||||
})
|
||||
return requestJson<MemoryGraphEdgeDetailPayload>(`/graph/edge-detail?${params.toString()}`)
|
||||
}
|
||||
|
||||
export async function previewMemoryDelete(
|
||||
payload: MemoryDeleteRequestPayload,
|
||||
): Promise<MemoryDeletePreviewPayload> {
|
||||
return requestJson<MemoryDeletePreviewPayload>('/delete/preview', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
}
|
||||
|
||||
export async function executeMemoryDelete(
|
||||
payload: MemoryDeleteRequestPayload,
|
||||
): Promise<MemoryDeleteExecutePayload> {
|
||||
return requestJson<MemoryDeleteExecutePayload>('/delete/execute', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
}
|
||||
|
||||
export async function restoreMemoryDelete(payload: {
|
||||
operation_id: string
|
||||
mode?: string
|
||||
selector?: Record<string, unknown> | string
|
||||
reason?: string
|
||||
requested_by?: string
|
||||
}): Promise<Record<string, unknown>> {
|
||||
return requestJson('/delete/restore', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
}
|
||||
|
||||
export async function getMemoryDeleteOperations(
|
||||
limit: number = 20,
|
||||
mode: string = '',
|
||||
): Promise<MemoryDeleteOperationListPayload> {
|
||||
const params = new URLSearchParams({ limit: String(limit) })
|
||||
if (mode.trim()) {
|
||||
params.set('mode', mode)
|
||||
}
|
||||
return requestJson<MemoryDeleteOperationListPayload>(`/delete/operations?${params.toString()}`)
|
||||
}
|
||||
|
||||
export async function getMemoryDeleteOperation(
|
||||
operationId: string,
|
||||
): Promise<MemoryDeleteOperationDetailPayload> {
|
||||
return requestJson<MemoryDeleteOperationDetailPayload>(`/delete/operations/${encodeURIComponent(operationId)}`)
|
||||
}
|
||||
|
||||
export async function getMemorySources(): Promise<MemorySourceListPayload> {
|
||||
return requestJson<MemorySourceListPayload>('/sources')
|
||||
}
|
||||
|
||||
export async function getMemoryRuntimeConfig(): Promise<MemoryRuntimeConfigPayload> {
|
||||
return requestJson<MemoryRuntimeConfigPayload>('/runtime/config')
|
||||
}
|
||||
|
||||
export async function refreshMemoryRuntimeSelfCheck(): Promise<MemoryRuntimeSelfCheckPayload> {
|
||||
return requestJson<MemoryRuntimeSelfCheckPayload>('/runtime/self-check/refresh', {
|
||||
method: 'POST',
|
||||
})
|
||||
}
|
||||
|
||||
export async function getMemoryConfigSchema(): Promise<MemoryConfigSchemaPayload> {
|
||||
return requestJson<MemoryConfigSchemaPayload>('/config/schema')
|
||||
}
|
||||
|
||||
export async function getMemoryConfig(): Promise<MemoryConfigPayload> {
|
||||
return requestJson<MemoryConfigPayload>('/config')
|
||||
}
|
||||
|
||||
export async function updateMemoryConfig(config: Record<string, unknown>): Promise<{ success: boolean; message?: string }> {
|
||||
return requestJson('/config', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ config }),
|
||||
})
|
||||
}
|
||||
|
||||
export async function getMemoryConfigRaw(): Promise<MemoryRawConfigPayload> {
|
||||
return requestJson<MemoryRawConfigPayload>('/config/raw')
|
||||
}
|
||||
|
||||
export async function updateMemoryConfigRaw(config: string): Promise<{ success: boolean; message?: string }> {
|
||||
return requestJson('/config/raw', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ config }),
|
||||
})
|
||||
}
|
||||
|
||||
export async function getMemoryImportGuide(): Promise<MemoryImportGuidePayload> {
|
||||
return requestJson<MemoryImportGuidePayload>('/import/guide')
|
||||
}
|
||||
|
||||
export async function getMemoryImportSettings(): Promise<Record<string, unknown>> {
|
||||
return requestJson('/import/settings')
|
||||
}
|
||||
|
||||
export async function getMemoryImportTasks(limit: number = 20): Promise<MemoryTaskListPayload> {
|
||||
return requestJson<MemoryTaskListPayload>(`/import/tasks?limit=${limit}`)
|
||||
}
|
||||
|
||||
export async function createMemoryPasteImport(payload: Record<string, unknown>): Promise<{ success: boolean; task?: MemoryTaskPayload }> {
|
||||
return requestJson('/import/paste', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
}
|
||||
|
||||
export async function getMemoryTuningProfile(): Promise<MemoryTuningProfilePayload> {
|
||||
return requestJson<MemoryTuningProfilePayload>('/retrieval_tuning/profile')
|
||||
}
|
||||
|
||||
export async function getMemoryTuningTasks(limit: number = 20): Promise<MemoryTaskListPayload> {
|
||||
return requestJson<MemoryTaskListPayload>(`/retrieval_tuning/tasks?limit=${limit}`)
|
||||
}
|
||||
|
||||
export async function createMemoryTuningTask(payload: Record<string, unknown>): Promise<{ success: boolean; task?: MemoryTaskPayload }> {
|
||||
return requestJson('/retrieval_tuning/tasks', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
}
|
||||
|
||||
export async function applyBestMemoryTuningProfile(taskId: string): Promise<{ success: boolean; error?: string }> {
|
||||
return requestJson(`/retrieval_tuning/tasks/${encodeURIComponent(taskId)}/apply-best`, {
|
||||
method: 'POST',
|
||||
})
|
||||
}
|
||||
|
||||
export async function getMemoryTuningReport(taskId: string, format: 'md' | 'json' = 'md'): Promise<{ success: boolean; content: string; path: string; error?: string }> {
|
||||
return requestJson(`/retrieval_tuning/tasks/${encodeURIComponent(taskId)}/report?format=${format}`)
|
||||
}
|
||||
80
dashboard/src/routes/__tests__/plugin-config.test.tsx
Normal file
80
dashboard/src/routes/__tests__/plugin-config.test.tsx
Normal file
@@ -0,0 +1,80 @@
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { ReactNode } from 'react'
|
||||
|
||||
import { PluginConfigPage } from '../plugin-config'
|
||||
import * as pluginApi from '@/lib/plugin-api'
|
||||
|
||||
const toastMock = vi.fn()
|
||||
|
||||
vi.mock('@/hooks/use-toast', () => ({
|
||||
useToast: () => ({ toast: toastMock }),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/restart-context', () => ({
|
||||
RestartProvider: ({ children }: { children: ReactNode }) => <>{children}</>,
|
||||
useRestart: () => ({
|
||||
showRestartPrompt: false,
|
||||
markRestartRequired: vi.fn(),
|
||||
clearRestartRequired: vi.fn(),
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('@/components/restart-overlay', () => ({
|
||||
RestartOverlay: () => null,
|
||||
}))
|
||||
|
||||
vi.mock('@/components', () => ({
|
||||
CodeEditor: ({ value }: { value: string }) => <pre>{value}</pre>,
|
||||
ListFieldEditor: () => <div>list-field-editor</div>,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/plugin-api', () => ({
|
||||
getInstalledPlugins: vi.fn(),
|
||||
getPluginConfigSchema: vi.fn(),
|
||||
getPluginConfig: vi.fn(),
|
||||
getPluginConfigRaw: vi.fn(),
|
||||
updatePluginConfig: vi.fn(),
|
||||
updatePluginConfigRaw: vi.fn(),
|
||||
resetPluginConfig: vi.fn(),
|
||||
togglePlugin: vi.fn(),
|
||||
}))
|
||||
|
||||
describe('PluginConfigPage', () => {
|
||||
beforeEach(() => {
|
||||
toastMock.mockReset()
|
||||
vi.mocked(pluginApi.getInstalledPlugins).mockResolvedValue({
|
||||
success: true,
|
||||
data: [
|
||||
{
|
||||
id: 'test.emoji',
|
||||
path: '/plugins/test_emoji',
|
||||
manifest: {
|
||||
manifest_version: 2,
|
||||
name: 'Emoji Plugin',
|
||||
version: '1.0.0',
|
||||
description: 'emoji tools',
|
||||
author: { name: 'tester' },
|
||||
license: 'MIT',
|
||||
host_application: { min_version: '1.0.0' },
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
vi.mocked(pluginApi.getPluginConfigSchema).mockResolvedValue({} as never)
|
||||
vi.mocked(pluginApi.getPluginConfig).mockResolvedValue({} as never)
|
||||
vi.mocked(pluginApi.getPluginConfigRaw).mockResolvedValue({} as never)
|
||||
vi.mocked(pluginApi.updatePluginConfig).mockResolvedValue({} as never)
|
||||
vi.mocked(pluginApi.updatePluginConfigRaw).mockResolvedValue({} as never)
|
||||
vi.mocked(pluginApi.resetPluginConfig).mockResolvedValue({} as never)
|
||||
vi.mocked(pluginApi.togglePlugin).mockResolvedValue({} as never)
|
||||
})
|
||||
|
||||
it('shows real plugins and no longer surfaces A_Memorix in plugin config list', async () => {
|
||||
render(<PluginConfigPage />)
|
||||
|
||||
expect(await screen.findByText('Emoji Plugin')).toBeInTheDocument()
|
||||
expect(screen.getByText('点击插件查看和编辑配置')).toBeInTheDocument()
|
||||
expect(screen.queryByText(/A_Memorix/i)).not.toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
@@ -76,6 +76,53 @@ interface FieldRendererProps {
|
||||
sectionName: string
|
||||
}
|
||||
|
||||
function getNestedRecord(config: Record<string, unknown>, path: string): Record<string, unknown> | undefined {
|
||||
const parts = path.split('.').filter(Boolean)
|
||||
let current: unknown = config
|
||||
|
||||
for (const part of parts) {
|
||||
if (!current || typeof current !== 'object' || Array.isArray(current)) {
|
||||
return undefined
|
||||
}
|
||||
current = (current as Record<string, unknown>)[part]
|
||||
}
|
||||
|
||||
if (!current || typeof current !== 'object' || Array.isArray(current)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return current as Record<string, unknown>
|
||||
}
|
||||
|
||||
function setNestedField(
|
||||
config: Record<string, unknown>,
|
||||
path: string,
|
||||
fieldName: string,
|
||||
value: unknown,
|
||||
): Record<string, unknown> {
|
||||
const parts = path.split('.').filter(Boolean)
|
||||
const nextConfig: Record<string, unknown> = { ...config }
|
||||
let currentTarget = nextConfig
|
||||
let currentSource: Record<string, unknown> | undefined = config
|
||||
|
||||
for (const part of parts) {
|
||||
const sourceValue: unknown = currentSource?.[part]
|
||||
const nextValue =
|
||||
sourceValue && typeof sourceValue === 'object' && !Array.isArray(sourceValue)
|
||||
? { ...(sourceValue as Record<string, unknown>) }
|
||||
: {}
|
||||
currentTarget[part] = nextValue
|
||||
currentTarget = nextValue
|
||||
currentSource =
|
||||
sourceValue && typeof sourceValue === 'object' && !Array.isArray(sourceValue)
|
||||
? (sourceValue as Record<string, unknown>)
|
||||
: undefined
|
||||
}
|
||||
|
||||
currentTarget[fieldName] = value
|
||||
return nextConfig
|
||||
}
|
||||
|
||||
function FieldRenderer({ field, value, onChange }: FieldRendererProps) {
|
||||
const [showPassword, setShowPassword] = useState(false)
|
||||
|
||||
@@ -91,7 +138,7 @@ function FieldRenderer({ field, value, onChange }: FieldRendererProps) {
|
||||
)}
|
||||
</div>
|
||||
<Switch
|
||||
checked={Boolean(value)}
|
||||
checked={Boolean(value ?? field.default)}
|
||||
onCheckedChange={onChange}
|
||||
disabled={field.disabled}
|
||||
/>
|
||||
@@ -222,7 +269,7 @@ function FieldRenderer({ field, value, onChange }: FieldRendererProps) {
|
||||
<div className="space-y-2">
|
||||
<Label>{field.label}</Label>
|
||||
<ListFieldEditor
|
||||
value={Array.isArray(value) ? value : []}
|
||||
value={Array.isArray(value) ? value : (Array.isArray(field.default) ? field.default : [])}
|
||||
onChange={(newValue) => onChange(newValue)}
|
||||
itemType={field.item_type ?? 'string'}
|
||||
itemFields={field.item_fields}
|
||||
@@ -267,6 +314,7 @@ interface SectionRendererProps {
|
||||
|
||||
function SectionRenderer({ section, config, onChange }: SectionRendererProps) {
|
||||
const [isOpen, setIsOpen] = useState(!section.collapsed)
|
||||
const sectionConfig = getNestedRecord(config, section.name)
|
||||
|
||||
// 按 order 排序字段
|
||||
const sortedFields = Object.entries(section.fields)
|
||||
@@ -304,7 +352,7 @@ function SectionRenderer({ section, config, onChange }: SectionRendererProps) {
|
||||
<FieldRenderer
|
||||
key={fieldName}
|
||||
field={field}
|
||||
value={(config[section.name] as Record<string, unknown>)?.[fieldName]}
|
||||
value={sectionConfig?.[fieldName]}
|
||||
onChange={(value) => onChange(section.name, fieldName, value)}
|
||||
sectionName={section.name}
|
||||
/>
|
||||
@@ -405,13 +453,7 @@ function PluginConfigEditor({ plugin, onBack }: PluginConfigEditorProps) {
|
||||
|
||||
// 处理字段变化
|
||||
const handleFieldChange = (sectionName: string, fieldName: string, value: unknown) => {
|
||||
setConfig(prev => ({
|
||||
...prev,
|
||||
[sectionName]: {
|
||||
...(prev[sectionName] as Record<string, unknown> || {}),
|
||||
[fieldName]: value
|
||||
}
|
||||
}))
|
||||
setConfig(prev => setNestedField(prev, sectionName, fieldName, value))
|
||||
}
|
||||
|
||||
// 保存配置
|
||||
|
||||
270
dashboard/src/routes/resource/__tests__/knowledge-base.test.tsx
Normal file
270
dashboard/src/routes/resource/__tests__/knowledge-base.test.tsx
Normal file
@@ -0,0 +1,270 @@
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { KnowledgeBasePage } from '../knowledge-base'
|
||||
import * as memoryApi from '@/lib/memory-api'
|
||||
|
||||
const navigateMock = vi.fn()
|
||||
const toastMock = vi.fn()
|
||||
|
||||
vi.mock('@tanstack/react-router', () => ({
|
||||
useNavigate: () => navigateMock,
|
||||
}))
|
||||
|
||||
vi.mock('@/hooks/use-toast', () => ({
|
||||
useToast: () => ({ toast: toastMock }),
|
||||
}))
|
||||
|
||||
vi.mock('@/components', () => ({
|
||||
CodeEditor: ({ value }: { value: string }) => <pre data-testid="code-editor">{value}</pre>,
|
||||
MarkdownRenderer: ({ content }: { content: string }) => <div>{content}</div>,
|
||||
}))
|
||||
|
||||
vi.mock('@/components/memory/MemoryConfigEditor', () => ({
|
||||
MemoryConfigEditor: () => <div data-testid="memory-config-editor">memory-config-editor</div>,
|
||||
}))
|
||||
|
||||
vi.mock('@/components/memory/MemoryDeleteDialog', () => ({
|
||||
MemoryDeleteDialog: ({
|
||||
open,
|
||||
preview,
|
||||
}: {
|
||||
open: boolean
|
||||
preview?: { mode?: string; item_count?: number } | null
|
||||
}) => (
|
||||
open ? <div data-testid="memory-delete-dialog">{`delete:${preview?.mode ?? 'none'}:${preview?.item_count ?? 0}`}</div> : null
|
||||
),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/memory-api', () => ({
|
||||
getMemoryConfigSchema: vi.fn(),
|
||||
getMemoryConfig: vi.fn(),
|
||||
getMemoryConfigRaw: vi.fn(),
|
||||
getMemoryDeleteOperation: vi.fn(),
|
||||
getMemoryRuntimeConfig: vi.fn(),
|
||||
getMemoryImportGuide: vi.fn(),
|
||||
getMemoryImportTasks: vi.fn(),
|
||||
getMemoryTuningProfile: vi.fn(),
|
||||
getMemoryTuningTasks: vi.fn(),
|
||||
getMemorySources: vi.fn(),
|
||||
getMemoryDeleteOperations: vi.fn(),
|
||||
refreshMemoryRuntimeSelfCheck: vi.fn(),
|
||||
updateMemoryConfig: vi.fn(),
|
||||
updateMemoryConfigRaw: vi.fn(),
|
||||
createMemoryPasteImport: vi.fn(),
|
||||
createMemoryTuningTask: vi.fn(),
|
||||
applyBestMemoryTuningProfile: vi.fn(),
|
||||
previewMemoryDelete: vi.fn(),
|
||||
executeMemoryDelete: vi.fn(),
|
||||
restoreMemoryDelete: vi.fn(),
|
||||
}))
|
||||
|
||||
describe('KnowledgeBasePage', () => {
|
||||
beforeEach(() => {
|
||||
navigateMock.mockReset()
|
||||
toastMock.mockReset()
|
||||
|
||||
vi.mocked(memoryApi.getMemoryConfigSchema).mockResolvedValue({
|
||||
success: true,
|
||||
path: 'config/a_memorix.toml',
|
||||
schema: {
|
||||
plugin_id: 'a_memorix',
|
||||
plugin_info: {
|
||||
name: 'A_Memorix',
|
||||
version: '2.0.0',
|
||||
description: '长期记忆子系统',
|
||||
author: 'A_Dawn',
|
||||
},
|
||||
_note: 'raw-only 字段仍可通过 TOML 编辑',
|
||||
layout: {
|
||||
type: 'tabs',
|
||||
tabs: [{ id: 'basic', title: '基础', sections: ['plugin'], order: 1 }],
|
||||
},
|
||||
sections: {
|
||||
plugin: {
|
||||
name: 'plugin',
|
||||
title: '子系统状态',
|
||||
collapsed: false,
|
||||
order: 1,
|
||||
fields: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryConfig).mockResolvedValue({
|
||||
success: true,
|
||||
path: 'config/a_memorix.toml',
|
||||
config: { plugin: { enabled: true } },
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryConfigRaw).mockResolvedValue({
|
||||
success: true,
|
||||
path: 'config/a_memorix.toml',
|
||||
config: '[plugin]\nenabled = true\n',
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryRuntimeConfig).mockResolvedValue({
|
||||
success: true,
|
||||
config: { plugin: { enabled: true } },
|
||||
data_dir: 'data/plugins/a-dawn.a-memorix',
|
||||
embedding_dimension: 1024,
|
||||
auto_save: true,
|
||||
relation_vectors_enabled: false,
|
||||
runtime_ready: true,
|
||||
embedding_degraded: false,
|
||||
embedding_degraded_reason: '',
|
||||
embedding_degraded_since: null,
|
||||
embedding_last_check: null,
|
||||
paragraph_vector_backfill_pending: 2,
|
||||
paragraph_vector_backfill_running: 0,
|
||||
paragraph_vector_backfill_failed: 1,
|
||||
paragraph_vector_backfill_done: 3,
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryImportGuide).mockResolvedValue({
|
||||
success: true,
|
||||
content: '# 导入指南\n导入说明',
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryImportTasks).mockResolvedValue({
|
||||
success: true,
|
||||
items: [{ task_id: 'import-1', status: 'done', mode: 'text' }],
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryTuningProfile).mockResolvedValue({
|
||||
success: true,
|
||||
profile: { retrieval: { top_k: 10 } },
|
||||
toml: '[retrieval]\ntop_k = 10\n',
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryTuningTasks).mockResolvedValue({
|
||||
success: true,
|
||||
items: [{ task_id: 'tune-1', status: 'done' }],
|
||||
})
|
||||
vi.mocked(memoryApi.getMemorySources).mockResolvedValue({
|
||||
success: true,
|
||||
items: [
|
||||
{ source: 'demo-1', paragraph_count: 2, relation_count: 1 },
|
||||
{ source: 'demo-2', paragraph_count: 1, relation_count: 0 },
|
||||
],
|
||||
count: 2,
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryDeleteOperations).mockResolvedValue({
|
||||
success: true,
|
||||
items: [
|
||||
{
|
||||
operation_id: 'del-1',
|
||||
mode: 'source',
|
||||
status: 'executed',
|
||||
summary: { counts: { paragraphs: 2, relations: 1, sources: 1 } },
|
||||
},
|
||||
],
|
||||
count: 1,
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryDeleteOperation).mockResolvedValue({
|
||||
success: true,
|
||||
operation: {
|
||||
operation_id: 'del-1',
|
||||
mode: 'source',
|
||||
status: 'executed',
|
||||
selector: { sources: ['demo-1'] },
|
||||
summary: { counts: { paragraphs: 2, relations: 1, sources: 1 }, sources: ['demo-1'] },
|
||||
items: [
|
||||
{
|
||||
item_type: 'paragraph',
|
||||
item_hash: 'p-1',
|
||||
item_key: 'paragraph:p-1',
|
||||
payload: { paragraph: { source: 'demo-1', content: '这是用于测试删除详情展示的段落内容。' } },
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
vi.mocked(memoryApi.refreshMemoryRuntimeSelfCheck).mockResolvedValue({
|
||||
success: true,
|
||||
report: { ok: true },
|
||||
})
|
||||
vi.mocked(memoryApi.updateMemoryConfig).mockResolvedValue({
|
||||
success: true,
|
||||
config_path: 'config/a_memorix.toml',
|
||||
} as never)
|
||||
vi.mocked(memoryApi.updateMemoryConfigRaw).mockResolvedValue({
|
||||
success: true,
|
||||
config_path: 'config/a_memorix.toml',
|
||||
} as never)
|
||||
vi.mocked(memoryApi.createMemoryPasteImport).mockResolvedValue({ success: true } as never)
|
||||
vi.mocked(memoryApi.createMemoryTuningTask).mockResolvedValue({ success: true } as never)
|
||||
vi.mocked(memoryApi.applyBestMemoryTuningProfile).mockResolvedValue({ success: true } as never)
|
||||
vi.mocked(memoryApi.previewMemoryDelete).mockResolvedValue({
|
||||
success: true,
|
||||
mode: 'source',
|
||||
selector: { sources: ['demo-1'] },
|
||||
counts: { sources: 1, paragraphs: 2, relations: 1 },
|
||||
sources: ['demo-1'],
|
||||
items: [{ item_type: 'paragraph', item_hash: 'p-1', label: 'demo-1' }],
|
||||
item_count: 1,
|
||||
dry_run: true,
|
||||
} as never)
|
||||
vi.mocked(memoryApi.executeMemoryDelete).mockResolvedValue({
|
||||
success: true,
|
||||
mode: 'source',
|
||||
operation_id: 'del-2',
|
||||
counts: { sources: 1, paragraphs: 2, relations: 1 },
|
||||
sources: ['demo-1'],
|
||||
deleted_count: 4,
|
||||
deleted_entity_count: 0,
|
||||
deleted_relation_count: 1,
|
||||
deleted_paragraph_count: 2,
|
||||
deleted_source_count: 1,
|
||||
} as never)
|
||||
vi.mocked(memoryApi.restoreMemoryDelete).mockResolvedValue({ success: true } as never)
|
||||
})
|
||||
|
||||
it('renders long-term memory console and key tabs', async () => {
|
||||
const user = userEvent.setup()
|
||||
|
||||
render(<KnowledgeBasePage />)
|
||||
|
||||
expect(await screen.findByText('长期记忆控制台')).toBeInTheDocument()
|
||||
expect(screen.getByText(/config\/a_memorix\.toml/)).toBeInTheDocument()
|
||||
expect(screen.getByText('运行就绪')).toBeInTheDocument()
|
||||
|
||||
await user.click(screen.getByRole('tab', { name: '配置' }))
|
||||
expect(await screen.findByTestId('memory-config-editor')).toBeInTheDocument()
|
||||
|
||||
await user.click(screen.getByRole('tab', { name: '导入' }))
|
||||
expect(await screen.findByText(/导入说明/)).toBeInTheDocument()
|
||||
expect(screen.getByText('import-1')).toBeInTheDocument()
|
||||
|
||||
await user.click(screen.getByRole('tab', { name: '调优' }))
|
||||
expect(await screen.findByText('tune-1')).toBeInTheDocument()
|
||||
expect(screen.getByRole('button', { name: '应用最佳' })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows delete tab and opens source delete preview', async () => {
|
||||
const user = userEvent.setup()
|
||||
|
||||
render(<KnowledgeBasePage />)
|
||||
|
||||
expect(await screen.findByText('长期记忆控制台')).toBeInTheDocument()
|
||||
await user.click(screen.getByRole('tab', { name: '删除' }))
|
||||
|
||||
expect(await screen.findByText('来源批量删除')).toBeInTheDocument()
|
||||
expect(screen.getAllByText('demo-1').length).toBeGreaterThan(0)
|
||||
expect(screen.getAllByText('del-1').length).toBeGreaterThan(0)
|
||||
expect(screen.getByText('恢复这次删除')).toBeInTheDocument()
|
||||
|
||||
await user.click(screen.getAllByRole('checkbox')[0])
|
||||
await user.click(screen.getByRole('button', { name: '预览删除' }))
|
||||
|
||||
expect(await screen.findByTestId('memory-delete-dialog')).toHaveTextContent('delete:source:1')
|
||||
})
|
||||
|
||||
it('loads selected delete operation detail items from detail endpoint', async () => {
|
||||
const user = userEvent.setup()
|
||||
|
||||
render(<KnowledgeBasePage />)
|
||||
|
||||
expect(await screen.findByText('长期记忆控制台')).toBeInTheDocument()
|
||||
await user.click(screen.getByRole('tab', { name: '删除' }))
|
||||
|
||||
expect(await screen.findByText('删除操作恢复')).toBeInTheDocument()
|
||||
expect(await screen.findByText('paragraph')).toBeInTheDocument()
|
||||
expect(screen.getByText('p-1')).toBeInTheDocument()
|
||||
expect(screen.getByText('这是用于测试删除详情展示的段落内容。')).toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
341
dashboard/src/routes/resource/__tests__/knowledge-graph.test.tsx
Normal file
341
dashboard/src/routes/resource/__tests__/knowledge-graph.test.tsx
Normal file
@@ -0,0 +1,341 @@
|
||||
import { render, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { KnowledgeGraphPage } from '../knowledge-graph'
|
||||
import * as memoryApi from '@/lib/memory-api'
|
||||
|
||||
const navigateMock = vi.fn()
|
||||
const toastMock = vi.fn()
|
||||
|
||||
vi.mock('@tanstack/react-router', () => ({
|
||||
useNavigate: () => navigateMock,
|
||||
}))
|
||||
|
||||
vi.mock('@/hooks/use-toast', () => ({
|
||||
useToast: () => ({ toast: toastMock }),
|
||||
}))
|
||||
|
||||
vi.mock('@/components/memory/MemoryDeleteDialog', () => ({
|
||||
MemoryDeleteDialog: ({
|
||||
open,
|
||||
preview,
|
||||
}: {
|
||||
open: boolean
|
||||
preview?: { mode?: string; item_count?: number } | null
|
||||
}) => (
|
||||
open ? <div data-testid="memory-delete-dialog">{`delete:${preview?.mode ?? 'none'}:${preview?.item_count ?? 0}`}</div> : null
|
||||
),
|
||||
}))
|
||||
|
||||
vi.mock('../knowledge-graph/GraphVisualization', () => ({
|
||||
GraphVisualization: ({
|
||||
graphData,
|
||||
onNodeClick,
|
||||
onEdgeClick,
|
||||
}: {
|
||||
graphData: { nodes: Array<{ id: string }>; edges: Array<{ source: string; target: string }> }
|
||||
onNodeClick: (event: React.MouseEvent, node: { id: string }) => void
|
||||
onEdgeClick: (event: React.MouseEvent, edge: { source: string; target: string }) => void
|
||||
}) => (
|
||||
<div data-testid="graph-visualization">
|
||||
<div>{`nodes:${graphData.nodes.length},edges:${graphData.edges.length}`}</div>
|
||||
{graphData.nodes[0] ? (
|
||||
<button type="button" onClick={(event) => onNodeClick(event as never, { id: graphData.nodes[0].id })}>
|
||||
选择节点
|
||||
</button>
|
||||
) : null}
|
||||
{graphData.edges[0] ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={(event) =>
|
||||
onEdgeClick(event as never, {
|
||||
source: graphData.edges[0].source,
|
||||
target: graphData.edges[0].target,
|
||||
})}
|
||||
>
|
||||
选择边
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
),
|
||||
}))
|
||||
|
||||
vi.mock('../knowledge-graph/GraphDialogs', () => ({
|
||||
NodeDetailDialog: ({
|
||||
selectedNodeData,
|
||||
nodeDetail,
|
||||
onOpenEvidence,
|
||||
onDeleteEntity,
|
||||
}: {
|
||||
selectedNodeData: { id: string } | null
|
||||
nodeDetail: { relations?: Array<{ predicate: string }>; paragraphs?: Array<unknown> } | null
|
||||
onOpenEvidence?: () => void
|
||||
onDeleteEntity?: (options: { includeParagraphs: boolean }) => void
|
||||
}) => (
|
||||
selectedNodeData ? (
|
||||
<div data-testid="node-detail-dialog">
|
||||
<div>{`node:${selectedNodeData.id}`}</div>
|
||||
<div>{`relations:${nodeDetail?.relations?.[0]?.predicate ?? 'none'}`}</div>
|
||||
<div>{`paragraphs:${nodeDetail?.paragraphs?.length ?? 0}`}</div>
|
||||
<button type="button" onClick={onOpenEvidence}>切到证据视图</button>
|
||||
<button type="button" onClick={() => onDeleteEntity?.({ includeParagraphs: true })}>删除实体</button>
|
||||
</div>
|
||||
) : null
|
||||
),
|
||||
EdgeDetailDialog: ({
|
||||
selectedEdgeData,
|
||||
edgeDetail,
|
||||
onOpenEvidence,
|
||||
}: {
|
||||
selectedEdgeData: { source: { id: string }; target: { id: string } } | null
|
||||
edgeDetail: { edge?: { predicates?: string[] }; paragraphs?: Array<unknown> } | null
|
||||
onOpenEvidence?: () => void
|
||||
}) => (
|
||||
selectedEdgeData ? (
|
||||
<div data-testid="edge-detail-dialog">
|
||||
<div>{`edge:${selectedEdgeData.source.id}->${selectedEdgeData.target.id}`}</div>
|
||||
<div>{`predicates:${edgeDetail?.edge?.predicates?.join(',') ?? 'none'}`}</div>
|
||||
<div>{`paragraphs:${edgeDetail?.paragraphs?.length ?? 0}`}</div>
|
||||
<button type="button" onClick={onOpenEvidence}>切到证据视图</button>
|
||||
</div>
|
||||
) : null
|
||||
),
|
||||
RelationDetailDialog: () => null,
|
||||
ParagraphDetailDialog: () => null,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/memory-api', () => ({
|
||||
getMemoryGraph: vi.fn(),
|
||||
getMemoryGraphNodeDetail: vi.fn(),
|
||||
getMemoryGraphEdgeDetail: vi.fn(),
|
||||
previewMemoryDelete: vi.fn(),
|
||||
executeMemoryDelete: vi.fn(),
|
||||
restoreMemoryDelete: vi.fn(),
|
||||
}))
|
||||
|
||||
describe('KnowledgeGraphPage', () => {
|
||||
beforeEach(() => {
|
||||
navigateMock.mockReset()
|
||||
toastMock.mockReset()
|
||||
vi.mocked(memoryApi.getMemoryGraph).mockResolvedValue({
|
||||
success: true,
|
||||
nodes: [
|
||||
{ id: 'alpha', name: 'Alpha' },
|
||||
{ id: 'beta', name: 'Beta' },
|
||||
],
|
||||
edges: [
|
||||
{
|
||||
source: 'alpha',
|
||||
target: 'beta',
|
||||
weight: 1,
|
||||
predicates: ['关联'],
|
||||
relation_count: 1,
|
||||
evidence_count: 2,
|
||||
relation_hashes: ['rel-1'],
|
||||
label: '关联',
|
||||
},
|
||||
],
|
||||
total_nodes: 2,
|
||||
total_edges: 1,
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryGraphNodeDetail).mockResolvedValue({
|
||||
success: true,
|
||||
node: { id: 'alpha', type: 'entity', content: 'Alpha', hash: 'entity-1', appearance_count: 3 },
|
||||
relations: [
|
||||
{
|
||||
hash: 'rel-1',
|
||||
subject: 'alpha',
|
||||
predicate: '关联',
|
||||
object: 'beta',
|
||||
text: 'alpha 关联 beta',
|
||||
confidence: 0.9,
|
||||
paragraph_count: 1,
|
||||
paragraph_hashes: ['p-1'],
|
||||
source_paragraph: 'p-1',
|
||||
},
|
||||
],
|
||||
paragraphs: [
|
||||
{
|
||||
hash: 'p-1',
|
||||
content: 'Alpha 提到了 Beta',
|
||||
preview: 'Alpha 提到了 Beta',
|
||||
source: 'demo',
|
||||
entity_count: 2,
|
||||
relation_count: 1,
|
||||
entities: ['Alpha', 'Beta'],
|
||||
relations: ['alpha 关联 beta'],
|
||||
},
|
||||
],
|
||||
evidence_graph: {
|
||||
nodes: [
|
||||
{ id: 'entity:alpha', type: 'entity', content: 'Alpha' },
|
||||
{ id: 'relation:rel-1', type: 'relation', content: 'alpha 关联 beta' },
|
||||
{ id: 'paragraph:p-1', type: 'paragraph', content: 'Alpha 提到了 Beta' },
|
||||
],
|
||||
edges: [
|
||||
{ source: 'paragraph:p-1', target: 'entity:alpha', kind: 'mentions', label: '提及', weight: 1 },
|
||||
{ source: 'paragraph:p-1', target: 'relation:rel-1', kind: 'supports', label: '支撑', weight: 1 },
|
||||
],
|
||||
focus_entities: ['alpha'],
|
||||
},
|
||||
})
|
||||
vi.mocked(memoryApi.getMemoryGraphEdgeDetail).mockResolvedValue({
|
||||
success: true,
|
||||
edge: {
|
||||
source: 'alpha',
|
||||
target: 'beta',
|
||||
weight: 1,
|
||||
predicates: ['关联'],
|
||||
relation_count: 1,
|
||||
evidence_count: 1,
|
||||
relation_hashes: ['rel-1'],
|
||||
label: '关联',
|
||||
},
|
||||
relations: [
|
||||
{
|
||||
hash: 'rel-1',
|
||||
subject: 'alpha',
|
||||
predicate: '关联',
|
||||
object: 'beta',
|
||||
text: 'alpha 关联 beta',
|
||||
confidence: 0.9,
|
||||
paragraph_count: 1,
|
||||
paragraph_hashes: ['p-1'],
|
||||
source_paragraph: 'p-1',
|
||||
},
|
||||
],
|
||||
paragraphs: [
|
||||
{
|
||||
hash: 'p-1',
|
||||
content: 'Alpha 提到了 Beta',
|
||||
preview: 'Alpha 提到了 Beta',
|
||||
source: 'demo',
|
||||
entity_count: 2,
|
||||
relation_count: 1,
|
||||
entities: ['Alpha', 'Beta'],
|
||||
relations: ['alpha 关联 beta'],
|
||||
},
|
||||
],
|
||||
evidence_graph: {
|
||||
nodes: [
|
||||
{ id: 'entity:alpha', type: 'entity', content: 'Alpha' },
|
||||
{ id: 'entity:beta', type: 'entity', content: 'Beta' },
|
||||
{ id: 'relation:rel-1', type: 'relation', content: 'alpha 关联 beta' },
|
||||
],
|
||||
edges: [
|
||||
{ source: 'relation:rel-1', target: 'entity:alpha', kind: 'subject', label: '主语', weight: 1 },
|
||||
{ source: 'relation:rel-1', target: 'entity:beta', kind: 'object', label: '宾语', weight: 1 },
|
||||
],
|
||||
focus_entities: ['alpha', 'beta'],
|
||||
},
|
||||
})
|
||||
vi.mocked(memoryApi.previewMemoryDelete).mockResolvedValue({
|
||||
success: true,
|
||||
mode: 'mixed',
|
||||
selector: { entity_hashes: ['entity-1'] },
|
||||
counts: { entities: 1, relations: 1, paragraphs: 1 },
|
||||
sources: ['demo'],
|
||||
items: [{ item_type: 'entity', item_hash: 'entity-1', label: 'Alpha' }],
|
||||
item_count: 1,
|
||||
dry_run: true,
|
||||
} as never)
|
||||
vi.mocked(memoryApi.executeMemoryDelete).mockResolvedValue({
|
||||
success: true,
|
||||
mode: 'mixed',
|
||||
operation_id: 'del-1',
|
||||
counts: { entities: 1, relations: 1, paragraphs: 1 },
|
||||
sources: ['demo'],
|
||||
deleted_count: 3,
|
||||
deleted_entity_count: 1,
|
||||
deleted_relation_count: 1,
|
||||
deleted_paragraph_count: 1,
|
||||
deleted_source_count: 0,
|
||||
} as never)
|
||||
vi.mocked(memoryApi.restoreMemoryDelete).mockResolvedValue({ success: true } as never)
|
||||
})
|
||||
|
||||
it('renders graph summary and supports empty-result filtering', async () => {
|
||||
const user = userEvent.setup()
|
||||
|
||||
render(<KnowledgeGraphPage />)
|
||||
|
||||
expect(await screen.findByText('长期记忆图谱')).toBeInTheDocument()
|
||||
expect(screen.getByText(/总节点 2/)).toBeInTheDocument()
|
||||
expect(screen.getByTestId('graph-visualization')).toHaveTextContent('nodes:2,edges:1')
|
||||
|
||||
await user.type(screen.getByPlaceholderText('筛选实体名称、节点 ID 或边标签'), 'missing')
|
||||
expect(memoryApi.getMemoryGraph).toHaveBeenCalledTimes(1)
|
||||
await user.click(screen.getByRole('button', { name: '筛选' }))
|
||||
|
||||
expect(await screen.findByText('还没有可展示的长期记忆图谱')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows empty state when switching to evidence view without a selection', async () => {
|
||||
const user = userEvent.setup()
|
||||
|
||||
render(<KnowledgeGraphPage />)
|
||||
|
||||
expect(await screen.findByTestId('graph-visualization')).toBeInTheDocument()
|
||||
await user.click(screen.getByRole('tab', { name: '证据视图' }))
|
||||
|
||||
expect(await screen.findByText('证据视图还没有可展示的选择')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('closes node dialog when switching to evidence view and renders evidence graph', async () => {
|
||||
const user = userEvent.setup()
|
||||
|
||||
render(<KnowledgeGraphPage />)
|
||||
|
||||
await screen.findByTestId('graph-visualization')
|
||||
await user.click(screen.getByRole('button', { name: '选择节点' }))
|
||||
|
||||
expect(await screen.findByTestId('node-detail-dialog')).toHaveTextContent('relations:关联')
|
||||
expect(screen.getByTestId('node-detail-dialog')).toHaveTextContent('paragraphs:1')
|
||||
|
||||
await user.click(screen.getByRole('button', { name: '切到证据视图' }))
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByTestId('node-detail-dialog')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId('graph-visualization')).toHaveTextContent('nodes:3,edges:2')
|
||||
})
|
||||
})
|
||||
|
||||
it('loads edge detail with predicates and support paragraphs', async () => {
|
||||
const user = userEvent.setup()
|
||||
|
||||
render(<KnowledgeGraphPage />)
|
||||
|
||||
await screen.findByTestId('graph-visualization')
|
||||
await user.click(screen.getByRole('button', { name: '选择边' }))
|
||||
|
||||
expect(await screen.findByTestId('edge-detail-dialog')).toHaveTextContent('predicates:关联')
|
||||
expect(screen.getByTestId('edge-detail-dialog')).toHaveTextContent('paragraphs:1')
|
||||
|
||||
await user.click(screen.getByRole('button', { name: '切到证据视图' }))
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByTestId('edge-detail-dialog')).not.toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
|
||||
it('opens delete preview dialog from node detail', async () => {
|
||||
const user = userEvent.setup()
|
||||
|
||||
render(<KnowledgeGraphPage />)
|
||||
|
||||
await screen.findByTestId('graph-visualization')
|
||||
await user.click(screen.getByRole('button', { name: '选择节点' }))
|
||||
await screen.findByTestId('node-detail-dialog')
|
||||
|
||||
await user.click(screen.getByRole('button', { name: '删除实体' }))
|
||||
|
||||
await waitFor(() => {
|
||||
expect(memoryApi.previewMemoryDelete).toHaveBeenCalled()
|
||||
})
|
||||
expect(await screen.findByTestId('memory-delete-dialog')).toHaveTextContent('delete:mixed:1')
|
||||
})
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,10 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
|
||||
import { Trash2 } from 'lucide-react'
|
||||
|
||||
import { Badge } from '@/components/ui/badge'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Checkbox } from '@/components/ui/checkbox'
|
||||
import {
|
||||
Dialog,
|
||||
DialogBody,
|
||||
@@ -7,63 +12,204 @@ import {
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog'
|
||||
import type {
|
||||
MemoryEvidenceParagraphNodeMetadata,
|
||||
MemoryEvidenceRelationNodeMetadata,
|
||||
MemoryGraphEdgeDetailPayload,
|
||||
MemoryGraphNodeDetailPayload,
|
||||
MemoryGraphParagraphDetailPayload,
|
||||
MemoryGraphRelationDetailPayload,
|
||||
} from '@/lib/memory-api'
|
||||
|
||||
import type { GraphNode, SelectedEdgeData } from './types'
|
||||
|
||||
function formatTimestamp(value?: number | null): string {
|
||||
if (!value) {
|
||||
return '未知'
|
||||
}
|
||||
const date = new Date(Number(value) * 1000)
|
||||
if (Number.isNaN(date.getTime())) {
|
||||
return '未知'
|
||||
}
|
||||
return date.toLocaleString()
|
||||
}
|
||||
|
||||
function RelationList({
|
||||
items,
|
||||
onDeleteRelation,
|
||||
}: {
|
||||
items: MemoryGraphRelationDetailPayload[]
|
||||
onDeleteRelation?: (relation: MemoryGraphRelationDetailPayload) => void
|
||||
}) {
|
||||
if (items.length <= 0) {
|
||||
return <p className="text-sm text-muted-foreground">暂无可展示的关系语义。</p>
|
||||
}
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
{items.map((relation) => (
|
||||
<div key={relation.hash} className="rounded-lg border bg-muted/40 p-3">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge variant="outline">{relation.predicate || '未命名谓词'}</Badge>
|
||||
<span className="text-xs text-muted-foreground">证据段落 {relation.paragraph_count}</span>
|
||||
<span className="text-xs text-muted-foreground">置信度 {relation.confidence.toFixed(3)}</span>
|
||||
</div>
|
||||
{onDeleteRelation ? (
|
||||
<Button size="sm" variant="outline" onClick={() => onDeleteRelation(relation)}>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
删除关系
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
<p className="mt-2 text-sm font-medium">{relation.text}</p>
|
||||
<code className="mt-2 block break-all text-xs text-muted-foreground">{relation.hash}</code>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function ParagraphList({
|
||||
items,
|
||||
onDeleteParagraph,
|
||||
}: {
|
||||
items: MemoryGraphParagraphDetailPayload[]
|
||||
onDeleteParagraph?: (paragraph: MemoryGraphParagraphDetailPayload) => void
|
||||
}) {
|
||||
if (items.length <= 0) {
|
||||
return <p className="text-sm text-muted-foreground">暂无可展示的来源段落。</p>
|
||||
}
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
{items.map((paragraph) => (
|
||||
<div key={paragraph.hash} className="rounded-lg border bg-muted/40 p-3">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge variant="secondary">{paragraph.source || '未命名来源'}</Badge>
|
||||
<span className="text-xs text-muted-foreground">实体 {paragraph.entity_count}</span>
|
||||
<span className="text-xs text-muted-foreground">关系 {paragraph.relation_count}</span>
|
||||
<span className="text-xs text-muted-foreground">更新时间 {formatTimestamp(paragraph.updated_at)}</span>
|
||||
</div>
|
||||
{onDeleteParagraph ? (
|
||||
<Button size="sm" variant="outline" onClick={() => onDeleteParagraph(paragraph)}>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
删除段落
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
<p className="mt-2 whitespace-pre-wrap text-sm break-words">{paragraph.preview || paragraph.content}</p>
|
||||
{paragraph.entities.length > 0 && (
|
||||
<div className="mt-2 flex flex-wrap gap-1">
|
||||
{paragraph.entities.slice(0, 8).map((entity) => (
|
||||
<Badge key={`${paragraph.hash}-${entity}`} variant="outline" className="text-xs">
|
||||
{entity}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
interface NodeDetailDialogProps {
|
||||
open: boolean
|
||||
onOpenChange: (open: boolean) => void
|
||||
selectedNodeData: GraphNode | null
|
||||
nodeDetail: MemoryGraphNodeDetailPayload | null
|
||||
loading?: boolean
|
||||
onOpenEvidence?: () => void
|
||||
onDeleteEntity?: (options: { includeParagraphs: boolean }) => void
|
||||
onDeleteRelation?: (relation: MemoryGraphRelationDetailPayload) => void
|
||||
onDeleteParagraph?: (paragraph: MemoryGraphParagraphDetailPayload) => void
|
||||
}
|
||||
|
||||
export function NodeDetailDialog({ open, onOpenChange, selectedNodeData }: NodeDetailDialogProps) {
|
||||
export function NodeDetailDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
selectedNodeData,
|
||||
nodeDetail,
|
||||
loading = false,
|
||||
onOpenEvidence,
|
||||
onDeleteEntity,
|
||||
onDeleteRelation,
|
||||
onDeleteParagraph,
|
||||
}: NodeDetailDialogProps) {
|
||||
const node = nodeDetail?.node ?? selectedNodeData
|
||||
const [includeParagraphs, setIncludeParagraphs] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) {
|
||||
setIncludeParagraphs(false)
|
||||
}
|
||||
}, [open, node?.id])
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="max-w-2xl max-h-[80vh] grid grid-rows-[auto_1fr_auto] overflow-hidden">
|
||||
<DialogContent className="max-w-4xl max-h-[85vh] grid grid-rows-[auto_1fr_auto] overflow-hidden">
|
||||
<DialogHeader>
|
||||
<DialogTitle>节点详情</DialogTitle>
|
||||
<DialogTitle>实体详情</DialogTitle>
|
||||
</DialogHeader>
|
||||
{selectedNodeData && (
|
||||
<DialogBody className="h-full">
|
||||
<div className="space-y-4 pb-2">
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<DialogBody className="h-full overflow-y-auto">
|
||||
{node ? (
|
||||
<div className="space-y-6 pb-2">
|
||||
<div className="flex flex-wrap items-start justify-between gap-3 rounded-xl border bg-muted/30 p-4">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-muted-foreground">类型</p>
|
||||
<div className="mt-1">
|
||||
<Badge variant={selectedNodeData.type === 'entity' ? 'default' : 'secondary'}>
|
||||
{selectedNodeData.type === 'entity' ? '🏷️ 实体' : '📄 段落'}
|
||||
</Badge>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge>{node.type === 'entity' ? '实体' : node.type}</Badge>
|
||||
{'appearance_count' in (nodeDetail?.node ?? {}) && (
|
||||
<Badge variant="outline">出现次数 {nodeDetail?.node.appearance_count ?? 0}</Badge>
|
||||
)}
|
||||
</div>
|
||||
<h3 className="mt-2 text-lg font-semibold">{node.content}</h3>
|
||||
<code className="mt-2 block break-all text-xs text-muted-foreground">{node.id}</code>
|
||||
</div>
|
||||
<div className="flex flex-col items-end gap-3">
|
||||
<Button variant="outline" onClick={onOpenEvidence} disabled={!onOpenEvidence}>
|
||||
切到证据视图
|
||||
</Button>
|
||||
{onDeleteEntity ? (
|
||||
<div className="flex flex-col items-end gap-2 rounded-lg border bg-background p-3">
|
||||
<label className="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
<Checkbox checked={includeParagraphs} onCheckedChange={(checked) => setIncludeParagraphs(Boolean(checked))} />
|
||||
删除该实体相关证据段落
|
||||
</label>
|
||||
<Button variant="outline" onClick={() => onDeleteEntity({ includeParagraphs })}>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
删除实体
|
||||
</Button>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<p className="text-sm font-medium text-muted-foreground">ID</p>
|
||||
<code className="mt-1 block p-2 bg-muted rounded text-xs break-all">
|
||||
{selectedNodeData.id}
|
||||
</code>
|
||||
</div>
|
||||
{loading ? (
|
||||
<p className="text-sm text-muted-foreground">正在加载节点证据…</p>
|
||||
) : (
|
||||
<>
|
||||
<section className="space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<h4 className="text-sm font-semibold">相关关系</h4>
|
||||
<span className="text-xs text-muted-foreground">{nodeDetail?.relations.length ?? 0} 条</span>
|
||||
</div>
|
||||
<RelationList items={nodeDetail?.relations ?? []} onDeleteRelation={onDeleteRelation} />
|
||||
</section>
|
||||
|
||||
<div>
|
||||
<p className="text-sm font-medium text-muted-foreground">内容</p>
|
||||
<div className="mt-1 p-3 bg-muted rounded border">
|
||||
<p className="text-sm whitespace-pre-wrap break-words">{selectedNodeData.content}</p>
|
||||
</div>
|
||||
{selectedNodeData.type === 'paragraph' && selectedNodeData.content && selectedNodeData.content.length < 20 && (
|
||||
<div className="mt-2 p-3 bg-yellow-50 dark:bg-yellow-950 border border-yellow-200 dark:border-yellow-800 rounded">
|
||||
<p className="text-xs text-yellow-800 dark:text-yellow-200">
|
||||
💡 <strong>提示：</strong>段落内容显示不完整？
|
||||
<br />
|
||||
您可以在 <strong>配置 → WebUI 服务配置</strong> 中启用 "在知识图谱中加载段落完整内容" 选项，以显示段落的完整文本。
|
||||
<br />
|
||||
注意：此功能会额外再次加载 embedding store，占用约数百MB内存。不建议在生产环境中长期开启。
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<section className="space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<h4 className="text-sm font-semibold">支持段落</h4>
|
||||
<span className="text-xs text-muted-foreground">{nodeDetail?.paragraphs.length ?? 0} 个</span>
|
||||
</div>
|
||||
<ParagraphList items={nodeDetail?.paragraphs ?? []} onDeleteParagraph={onDeleteParagraph} />
|
||||
</section>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</DialogBody>
|
||||
)}
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">尚未选中实体。</p>
|
||||
)}
|
||||
</DialogBody>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
@@ -73,49 +219,226 @@ interface EdgeDetailDialogProps {
|
||||
open: boolean
|
||||
onOpenChange: (open: boolean) => void
|
||||
selectedEdgeData: SelectedEdgeData | null
|
||||
edgeDetail: MemoryGraphEdgeDetailPayload | null
|
||||
loading?: boolean
|
||||
onOpenEvidence?: () => void
|
||||
onDeleteEdgeGroup?: (options: { includeParagraphs: boolean }) => void
|
||||
onDeleteRelation?: (relation: MemoryGraphRelationDetailPayload) => void
|
||||
onDeleteParagraph?: (paragraph: MemoryGraphParagraphDetailPayload) => void
|
||||
}
|
||||
|
||||
export function EdgeDetailDialog({ open, onOpenChange, selectedEdgeData }: EdgeDetailDialogProps) {
|
||||
export function EdgeDetailDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
selectedEdgeData,
|
||||
edgeDetail,
|
||||
loading = false,
|
||||
onOpenEvidence,
|
||||
onDeleteEdgeGroup,
|
||||
onDeleteRelation,
|
||||
onDeleteParagraph,
|
||||
}: EdgeDetailDialogProps) {
|
||||
const sourceLabel = selectedEdgeData?.source.content ?? edgeDetail?.edge.source ?? ''
|
||||
const targetLabel = selectedEdgeData?.target.content ?? edgeDetail?.edge.target ?? ''
|
||||
const [includeParagraphs, setIncludeParagraphs] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) {
|
||||
setIncludeParagraphs(false)
|
||||
}
|
||||
}, [open, edgeDetail?.edge.source, edgeDetail?.edge.target])
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="max-w-2xl max-h-[80vh] overflow-hidden flex flex-col">
|
||||
<DialogContent className="max-w-4xl max-h-[85vh] overflow-hidden grid grid-rows-[auto_1fr_auto]">
|
||||
<DialogHeader>
|
||||
<DialogTitle>边详情</DialogTitle>
|
||||
<DialogTitle>关系详情</DialogTitle>
|
||||
</DialogHeader>
|
||||
{selectedEdgeData && (
|
||||
<DialogBody>
|
||||
<div className="space-y-4">
|
||||
<div className="flex items-center gap-4">
|
||||
<div className="flex-1 min-w-0 p-3 bg-blue-50 dark:bg-blue-950 rounded border-2 border-blue-200 dark:border-blue-800">
|
||||
<div className="text-xs text-muted-foreground mb-1">源节点</div>
|
||||
<div className="font-medium text-sm mb-2 truncate">{selectedEdgeData.source.content}</div>
|
||||
<code className="text-xs text-muted-foreground truncate block">
|
||||
{selectedEdgeData.source.id.slice(0, 40)}...
|
||||
</code>
|
||||
</div>
|
||||
|
||||
<div className="text-2xl text-muted-foreground flex-shrink-0">→</div>
|
||||
|
||||
<div className="flex-1 min-w-0 p-3 bg-green-50 dark:bg-green-950 rounded border-2 border-green-200 dark:border-green-800">
|
||||
<div className="text-xs text-muted-foreground mb-1">目标节点</div>
|
||||
<div className="font-medium text-sm mb-2 truncate">{selectedEdgeData.target.content}</div>
|
||||
<code className="text-xs text-muted-foreground truncate block">
|
||||
{selectedEdgeData.target.id.slice(0, 40)}...
|
||||
</code>
|
||||
<DialogBody className="overflow-y-auto">
|
||||
{selectedEdgeData || edgeDetail ? (
|
||||
<div className="space-y-6 pb-2">
|
||||
<div className="rounded-xl border bg-muted/30 p-4">
|
||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
{(edgeDetail?.edge.predicates ?? []).map((predicate) => (
|
||||
<Badge key={predicate} variant="outline">{predicate}</Badge>
|
||||
))}
|
||||
<Badge variant="secondary">关系 {edgeDetail?.edge.relation_count ?? selectedEdgeData?.edge.relationCount ?? 0}</Badge>
|
||||
<Badge variant="secondary">证据 {edgeDetail?.edge.evidence_count ?? selectedEdgeData?.edge.evidenceCount ?? 0}</Badge>
|
||||
</div>
|
||||
<p className="mt-3 text-base font-semibold break-words">
|
||||
{sourceLabel} → {targetLabel}
|
||||
</p>
|
||||
<p className="mt-2 text-sm text-muted-foreground">
|
||||
聚合权重 {(edgeDetail?.edge.weight ?? selectedEdgeData?.edge.weight ?? 0).toFixed(4)}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-col items-end gap-3">
|
||||
<Button variant="outline" onClick={onOpenEvidence} disabled={!onOpenEvidence}>
|
||||
切到证据视图
|
||||
</Button>
|
||||
{onDeleteEdgeGroup ? (
|
||||
<div className="flex flex-col items-end gap-2 rounded-lg border bg-background p-3">
|
||||
<label className="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
<Checkbox checked={includeParagraphs} onCheckedChange={(checked) => setIncludeParagraphs(Boolean(checked))} />
|
||||
同时删除支撑段落
|
||||
</label>
|
||||
<Button variant="outline" onClick={() => onDeleteEdgeGroup({ includeParagraphs })}>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
删除此关系组
|
||||
</Button>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<p className="text-sm font-medium text-muted-foreground">权重</p>
|
||||
<div className="mt-1">
|
||||
<Badge variant="outline" className="text-base font-mono">
|
||||
{selectedEdgeData.edge.weight.toFixed(4)}
|
||||
</Badge>
|
||||
</div>
|
||||
</div>
|
||||
{loading ? (
|
||||
<p className="text-sm text-muted-foreground">正在加载边的证据…</p>
|
||||
) : (
|
||||
<>
|
||||
<section className="space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<h4 className="text-sm font-semibold">关系语义</h4>
|
||||
<span className="text-xs text-muted-foreground">{edgeDetail?.relations.length ?? 0} 条</span>
|
||||
</div>
|
||||
<RelationList items={edgeDetail?.relations ?? []} onDeleteRelation={onDeleteRelation} />
|
||||
</section>
|
||||
|
||||
<section className="space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<h4 className="text-sm font-semibold">支持段落</h4>
|
||||
<span className="text-xs text-muted-foreground">{edgeDetail?.paragraphs.length ?? 0} 个</span>
|
||||
</div>
|
||||
<ParagraphList items={edgeDetail?.paragraphs ?? []} onDeleteParagraph={onDeleteParagraph} />
|
||||
</section>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</DialogBody>
|
||||
)}
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">尚未选中关系。</p>
|
||||
)}
|
||||
</DialogBody>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
|
||||
interface RelationDetailDialogProps {
|
||||
open: boolean
|
||||
onOpenChange: (open: boolean) => void
|
||||
relation: MemoryGraphRelationDetailPayload | null
|
||||
metadata?: MemoryEvidenceRelationNodeMetadata | null
|
||||
onDeleteRelation?: (relation: MemoryGraphRelationDetailPayload, includeParagraphs: boolean) => void
|
||||
}
|
||||
|
||||
export function RelationDetailDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
relation,
|
||||
metadata,
|
||||
onDeleteRelation,
|
||||
}: RelationDetailDialogProps) {
|
||||
const [includeParagraphs, setIncludeParagraphs] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) {
|
||||
setIncludeParagraphs(false)
|
||||
}
|
||||
}, [open, relation?.hash])
|
||||
|
||||
if (!relation) {
|
||||
return null
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="max-w-2xl max-h-[80vh] grid grid-rows-[auto_1fr_auto] overflow-hidden">
|
||||
<DialogHeader>
|
||||
<DialogTitle>关系明细</DialogTitle>
|
||||
</DialogHeader>
|
||||
<DialogBody className="space-y-4 overflow-y-auto">
|
||||
<div className="rounded-xl border bg-muted/30 p-4">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge variant="outline">{relation.predicate || metadata?.predicate || '未命名谓词'}</Badge>
|
||||
<Badge variant="secondary">证据段落 {relation.paragraph_count}</Badge>
|
||||
<Badge variant="secondary">置信度 {relation.confidence.toFixed(3)}</Badge>
|
||||
</div>
|
||||
<p className="mt-3 text-base font-semibold break-words">{relation.text}</p>
|
||||
<code className="mt-3 block break-all text-xs text-muted-foreground">{relation.hash}</code>
|
||||
</div>
|
||||
|
||||
{onDeleteRelation ? (
|
||||
<div className="rounded-lg border bg-background p-3">
|
||||
<label className="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
<Checkbox checked={includeParagraphs} onCheckedChange={(checked) => setIncludeParagraphs(Boolean(checked))} />
|
||||
同时删除支撑该关系的段落
|
||||
</label>
|
||||
<Button className="mt-3" variant="outline" onClick={() => onDeleteRelation(relation, includeParagraphs)}>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
删除这条关系
|
||||
</Button>
|
||||
</div>
|
||||
) : null}
|
||||
</DialogBody>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
|
||||
interface ParagraphDetailDialogProps {
|
||||
open: boolean
|
||||
onOpenChange: (open: boolean) => void
|
||||
paragraph: MemoryGraphParagraphDetailPayload | null
|
||||
metadata?: MemoryEvidenceParagraphNodeMetadata | null
|
||||
onDeleteParagraph?: (paragraph: MemoryGraphParagraphDetailPayload) => void
|
||||
}
|
||||
|
||||
export function ParagraphDetailDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
paragraph,
|
||||
metadata,
|
||||
onDeleteParagraph,
|
||||
}: ParagraphDetailDialogProps) {
|
||||
if (!paragraph) {
|
||||
return null
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="max-w-2xl max-h-[80vh] grid grid-rows-[auto_1fr_auto] overflow-hidden">
|
||||
<DialogHeader>
|
||||
<DialogTitle>段落明细</DialogTitle>
|
||||
</DialogHeader>
|
||||
<DialogBody className="space-y-4 overflow-y-auto">
|
||||
<div className="rounded-xl border bg-muted/30 p-4">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge variant="secondary">{paragraph.source || metadata?.source || '未命名来源'}</Badge>
|
||||
<Badge variant="outline">实体 {paragraph.entity_count}</Badge>
|
||||
<Badge variant="outline">关系 {paragraph.relation_count}</Badge>
|
||||
<Badge variant="outline">更新时间 {formatTimestamp(paragraph.updated_at ?? metadata?.updated_at)}</Badge>
|
||||
</div>
|
||||
<p className="mt-3 whitespace-pre-wrap text-sm break-words">{paragraph.content}</p>
|
||||
<code className="mt-3 block break-all text-xs text-muted-foreground">{paragraph.hash}</code>
|
||||
</div>
|
||||
|
||||
{paragraph.entities.length > 0 ? (
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{paragraph.entities.map((entity) => (
|
||||
<Badge key={`${paragraph.hash}-${entity}`} variant="outline">{entity}</Badge>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{onDeleteParagraph ? (
|
||||
<Button variant="outline" onClick={() => onDeleteParagraph(paragraph)}>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
删除这段证据
|
||||
</Button>
|
||||
) : null}
|
||||
</DialogBody>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { memo, useCallback } from 'react'
|
||||
import { memo, useCallback, useMemo } from 'react'
|
||||
import ReactFlow, {
|
||||
Background,
|
||||
BackgroundVariant,
|
||||
@@ -7,8 +7,6 @@ import ReactFlow, {
|
||||
MiniMap,
|
||||
Panel,
|
||||
Position,
|
||||
useEdgesState,
|
||||
useNodesState,
|
||||
type Edge,
|
||||
type Node,
|
||||
type NodeTypes,
|
||||
@@ -47,8 +45,23 @@ const ParagraphNode = memo(({ data }: { data: { label: string; content: string }
|
||||
|
||||
ParagraphNode.displayName = 'ParagraphNode'
|
||||
|
||||
const RelationNode = memo(({ data }: { data: { label: string; content: string } }) => {
|
||||
return (
|
||||
<div className="px-3 py-2 shadow-md rounded-md bg-gradient-to-br from-amber-500 to-orange-600 border-2 border-orange-700 min-w-[140px]">
|
||||
<Handle type="target" position={Position.Top} />
|
||||
<div className="font-medium text-white text-xs truncate max-w-[180px]" title={data.content}>
|
||||
{data.label}
|
||||
</div>
|
||||
<Handle type="source" position={Position.Bottom} />
|
||||
</div>
|
||||
)
|
||||
})
|
||||
|
||||
RelationNode.displayName = 'RelationNode'
|
||||
|
||||
const nodeTypes: NodeTypes = {
|
||||
entity: EntityNode,
|
||||
relation: RelationNode,
|
||||
paragraph: ParagraphNode,
|
||||
}
|
||||
|
||||
@@ -61,7 +74,13 @@ function calculateLayout(nodes: GraphNode[], edges: GraphEdge[]): { nodes: FlowN
|
||||
const flowEdges: FlowEdge[] = []
|
||||
|
||||
nodes.forEach((node) => {
|
||||
dagreGraph.setNode(node.id, { width: 150, height: 50 })
|
||||
const size =
|
||||
node.type === 'relation'
|
||||
? { width: 180, height: 60 }
|
||||
: node.type === 'paragraph'
|
||||
? { width: 190, height: 56 }
|
||||
: { width: 150, height: 50 }
|
||||
dagreGraph.setNode(node.id, size)
|
||||
})
|
||||
|
||||
edges.forEach((edge) => {
|
||||
@@ -82,22 +101,45 @@ function calculateLayout(nodes: GraphNode[], edges: GraphEdge[]): { nodes: FlowN
|
||||
data: {
|
||||
label: node.content.slice(0, 20) + (node.content.length > 20 ? '...' : ''),
|
||||
content: node.content,
|
||||
type: node.type,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
edges.forEach((edge, index) => {
|
||||
const isEvidenceEdge = edge.kind && edge.kind !== 'relation'
|
||||
const strokeColor =
|
||||
edge.kind === 'mentions'
|
||||
? '#0f766e'
|
||||
: edge.kind === 'supports'
|
||||
? '#b45309'
|
||||
: edge.kind === 'subject'
|
||||
? '#4f46e5'
|
||||
: edge.kind === 'object'
|
||||
? '#7c3aed'
|
||||
: '#64748b'
|
||||
const flowEdge: FlowEdge = {
|
||||
id: `edge-${index}`,
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
animated: nodes.length <= 200 && edge.weight > 5,
|
||||
animated: nodes.length <= 200 && (isEvidenceEdge || edge.weight > 5),
|
||||
style: {
|
||||
strokeWidth: Math.min(edge.weight / 2, 5),
|
||||
opacity: 0.6,
|
||||
strokeWidth: isEvidenceEdge ? Math.min(Math.max(edge.weight, 1.5), 4) : Math.min(edge.weight / 2, 5),
|
||||
opacity: isEvidenceEdge ? 0.9 : 0.6,
|
||||
stroke: strokeColor,
|
||||
},
|
||||
labelStyle: {
|
||||
fill: '#334155',
|
||||
fontSize: 11,
|
||||
fontWeight: 600,
|
||||
},
|
||||
labelBgPadding: [6, 2],
|
||||
labelBgBorderRadius: 6,
|
||||
labelBgStyle: { fill: 'rgba(255,255,255,0.88)', fillOpacity: 0.95 },
|
||||
}
|
||||
if (edge.weight > 10 && nodes.length < 100) {
|
||||
if (edge.label && (isEvidenceEdge || nodes.length <= 120)) {
|
||||
flowEdge.label = edge.label
|
||||
} else if (edge.weight > 10 && nodes.length < 100) {
|
||||
flowEdge.label = `${edge.weight.toFixed(0)}`
|
||||
}
|
||||
flowEdges.push(flowEdge)
|
||||
@@ -114,13 +156,19 @@ interface GraphVisualizationProps {
|
||||
}
|
||||
|
||||
export function GraphVisualization({ graphData, onNodeClick, onEdgeClick, loading = false }: GraphVisualizationProps) {
|
||||
const { nodes: flowNodes, edges: flowEdges } = calculateLayout(graphData.nodes, graphData.edges)
|
||||
const [nodes, , onNodesChange] = useNodesState(flowNodes)
|
||||
const [edges, , onEdgesChange] = useEdgesState(flowEdges)
|
||||
const nodeCount = nodes.length
|
||||
const { nodes: flowNodes, edges: flowEdges } = useMemo(
|
||||
() => calculateLayout(graphData.nodes, graphData.edges),
|
||||
[graphData.edges, graphData.nodes],
|
||||
)
|
||||
const nodeCount = flowNodes.length
|
||||
const graphMode = useMemo(
|
||||
() => (graphData.nodes.some((node) => node.type !== 'entity') ? 'evidence' : 'entity'),
|
||||
[graphData.nodes],
|
||||
)
|
||||
|
||||
const miniMapNodeColor = useCallback((node: Node) => {
|
||||
if (node.type === 'entity') return '#6366f1'
|
||||
if (node.type === 'relation') return '#f59e0b'
|
||||
if (node.type === 'paragraph') return '#10b981'
|
||||
return '#6b7280'
|
||||
}, [])
|
||||
@@ -133,17 +181,15 @@ export function GraphVisualization({ graphData, onNodeClick, onEdgeClick, loadin
|
||||
<div
|
||||
style={{ touchAction: 'none' }}
|
||||
role="img"
|
||||
aria-label={`知识图谱可视化，共 ${nodeCount} 个节点，${edges.length} 条关系`}
|
||||
aria-label={`知识图谱可视化，共 ${nodeCount} 个节点，${flowEdges.length} 条关系`}
|
||||
className="w-full h-full"
|
||||
>
|
||||
<span className="sr-only">
|
||||
{`知识图谱包含 ${nodeCount} 个节点和 ${edges.length} 条关系。`}
|
||||
{`知识图谱包含 ${nodeCount} 个节点和 ${flowEdges.length} 条关系。`}
|
||||
</span>
|
||||
<ReactFlow
|
||||
nodes={nodes}
|
||||
edges={edges}
|
||||
onNodesChange={onNodesChange}
|
||||
onEdgesChange={onEdgesChange}
|
||||
nodes={flowNodes}
|
||||
edges={flowEdges}
|
||||
onNodeClick={onNodeClick}
|
||||
onEdgeClick={onEdgeClick}
|
||||
nodeTypes={nodeTypes}
|
||||
@@ -171,16 +217,34 @@ export function GraphVisualization({ graphData, onNodeClick, onEdgeClick, loadin
|
||||
)}
|
||||
|
||||
<Panel position="top-right" className="bg-background/95 backdrop-blur-sm rounded-lg border p-3 shadow-lg">
|
||||
<div className="text-sm font-semibold mb-2">图例</div>
|
||||
<div className="text-sm font-semibold mb-2">
|
||||
{graphMode === 'entity' ? '实体关系图图例' : '证据视图图例'}
|
||||
</div>
|
||||
<div className="space-y-2 text-xs">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-4 h-4 rounded bg-gradient-to-br from-blue-500 to-blue-600 border-2 border-blue-700" aria-hidden="true" />
|
||||
<span>实体节点</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-4 h-4 rounded bg-gradient-to-br from-green-500 to-green-600 border-2 border-green-700" aria-hidden="true" />
|
||||
<span>段落节点</span>
|
||||
</div>
|
||||
{graphMode === 'evidence' && (
|
||||
<>
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-4 h-4 rounded bg-gradient-to-br from-amber-500 to-orange-600 border-2 border-orange-700" aria-hidden="true" />
|
||||
<span>关系节点</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-4 h-4 rounded bg-gradient-to-br from-green-500 to-green-600 border-2 border-green-700" aria-hidden="true" />
|
||||
<span>段落节点</span>
|
||||
</div>
|
||||
<div className="text-muted-foreground">
|
||||
紫色线表示关系到宾语，蓝色线表示关系到主语，绿色/橙色线表示段落证据。
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{graphMode === 'entity' && (
|
||||
<div className="text-muted-foreground">
|
||||
线条表示实体间聚合关系，边标签优先显示主谓词，更多语义可点击查看详情。
|
||||
</div>
|
||||
)}
|
||||
{nodeCount > 200 && (
|
||||
<div className="mt-2 pt-2 border-t text-yellow-600 dark:text-yellow-500">
|
||||
<div className="font-semibold">性能模式</div>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,19 +2,27 @@ import type { Node, Edge } from 'reactflow'
|
||||
|
||||
export interface GraphNode {
|
||||
id: string
|
||||
type: 'entity' | 'paragraph'
|
||||
type: 'entity' | 'relation' | 'paragraph'
|
||||
content: string
|
||||
metadata?: Record<string, unknown>
|
||||
}
|
||||
|
||||
export interface GraphEdge {
|
||||
source: string
|
||||
target: string
|
||||
weight: number
|
||||
kind?: 'relation' | 'mentions' | 'supports' | 'subject' | 'object'
|
||||
label?: string
|
||||
relationHashes?: string[]
|
||||
predicates?: string[]
|
||||
relationCount?: number
|
||||
evidenceCount?: number
|
||||
}
|
||||
|
||||
export interface GraphData {
|
||||
nodes: GraphNode[]
|
||||
edges: GraphEdge[]
|
||||
focusEntities?: string[]
|
||||
}
|
||||
|
||||
export interface GraphStats {
|
||||
@@ -27,6 +35,7 @@ export interface GraphStats {
|
||||
export interface FlowNodeData {
|
||||
label: string
|
||||
content: string
|
||||
type: GraphNode['type']
|
||||
}
|
||||
|
||||
export type FlowNode = Node<FlowNodeData>
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
# A_Memorix 子模块维护说明（维护者内部文档）
|
||||
|
||||
> 本文档用于维护者，不面向普通用户。
|
||||
|
||||
## 1. 基本约束
|
||||
- 子模块路径固定：`plugins/A_memorix`
|
||||
- 子模块仓库固定：`https://github.com/A-Dawn/A_memorix.git`
|
||||
- 子模块分支固定：`MaiBot_branch`
|
||||
- 强约束：主仓内 `plugins/A_memorix` 指针必须等于远端 `origin/MaiBot_branch` 最新 HEAD
|
||||
|
||||
## 2. 首次拉取/恢复子模块
|
||||
```bash
|
||||
git submodule update --init --recursive
|
||||
```
|
||||
|
||||
若目录为空或缺少 `_manifest.json`，先执行上面的命令再排查其他问题。
|
||||
|
||||
## 3. 维护者更新流程
|
||||
1. 先在外部仓 `MaiBot_branch` 完成目标功能合入。
|
||||
2. 在主仓执行：
|
||||
```bash
|
||||
git submodule update --remote --recursive plugins/A_memorix
|
||||
git add plugins/A_memorix .gitmodules
|
||||
git commit -m "chore(submodule): bump A_memorix"
|
||||
```
|
||||
|
||||
## 4. CI 严格校验说明
|
||||
- PR Precheck 会校验：
|
||||
- `.gitmodules` 的 path/url/branch 必须匹配固定值
|
||||
- 子模块指针必须等于远端 `MaiBot_branch` 最新 HEAD
|
||||
- Docker 构建工作流在构建前也会执行同样的 fail-fast 对齐检查
|
||||
|
||||
## 5. 回滚策略
|
||||
- 回滚主仓提交会同时回滚子模块指针。
|
||||
- 但若回滚后的指针不再是远端 `MaiBot_branch` 最新 HEAD，CI 会阻断。
|
||||
- 处理方式：
|
||||
- 先在外部仓移动/回滚 `MaiBot_branch` 到目标提交，再重跑；
|
||||
- 或按团队流程申请一次性 CI 豁免。
|
||||
45
docs/a_memorix_sync.md
Normal file
45
docs/a_memorix_sync.md
Normal file
@@ -0,0 +1,45 @@
|
||||
# A_Memorix 同步说明
|
||||
|
||||
## 当前约定
|
||||
|
||||
- A_Memorix 主线源码位于 `src/A_memorix`
|
||||
- 宿主接入层位于 `src/services/memory_service.py`、`src/webui/routers/memory.py` 与 dashboard 长期记忆页面
|
||||
- 运行配置位于 `config/a_memorix.toml`
|
||||
- 运行数据位于 `data/plugins/a-dawn.a-memorix/`
|
||||
- 上游同步方式固定为 `git subtree`
|
||||
|
||||
## 首次接入
|
||||
|
||||
```bash
|
||||
./scripts/sync_a_memorix_subtree.sh add
|
||||
```
|
||||
|
||||
默认同步源：
|
||||
|
||||
- 远端：`https://github.com/A-Dawn/A_memorix.git`
|
||||
- 分支：`MaiBot_branch`
|
||||
- 前缀：`src/A_memorix`
|
||||
|
||||
## 后续更新
|
||||
|
||||
```bash
|
||||
./scripts/sync_a_memorix_subtree.sh pull
|
||||
```
|
||||
|
||||
等价命令：
|
||||
|
||||
```bash
|
||||
git subtree pull --prefix=src/A_memorix https://github.com/A-Dawn/A_memorix.git MaiBot_branch --squash
|
||||
```
|
||||
|
||||
## 本地修改边界
|
||||
|
||||
- `src/A_memorix` 只保留上游源码和必须的宿主兼容补丁
|
||||
- 宿主接入、配置暴露、图谱页与控制台页优先放在 MaiBot 侧 host service / memory router / dashboard 页面中
|
||||
- 不再通过插件框架特判承载 A_Memorix
|
||||
|
||||
## 同步后检查
|
||||
|
||||
1. 确认 `config/a_memorix.toml` 仍指向 `data/plugins/a-dawn.a-memorix`
|
||||
2. 运行 `python src/A_memorix/scripts/runtime_self_check.py --help`
|
||||
3. 运行 A_memorix 相关测试或最少执行一次针对性导入验证
|
||||
Submodule plugins/A_memorix deleted from 60224ce275
65
pytests/A_memorix_test/data/benchmarks/README.md
Normal file
65
pytests/A_memorix_test/data/benchmarks/README.md
Normal file
@@ -0,0 +1,65 @@
|
||||
# Group Chat Stream Benchmark
|
||||
|
||||
这套基准数据专门用于 A_memorix 当前“群聊聊天流”设计的量化评估。
|
||||
|
||||
设计对齐点：
|
||||
|
||||
- 只把 Bot 参与过的话题段落纳入长期记忆总结。
|
||||
- 群聊内容先按话题批次收束，再写入 `chat_summary`。
|
||||
- 回复前通过 `search_long_term_memory` 做长期记忆检索增强。
|
||||
- 回复后只把“关于人物的稳定事实”写回 `person_fact`。
|
||||
- 检索需要覆盖 `search / time / episode / aggregate` 四种模式。
|
||||
- 需要有明确的负样本，验证“无 Bot 参与”的纯群友闲聊不会被误写入。
|
||||
- 当前 summarizer 的原生触发条件需要被显式覆盖：
|
||||
`80` 条消息直接触发，或 `8` 小时后累计至少 `20` 条消息触发。
|
||||
|
||||
数据文件：
|
||||
|
||||
- `group_chat_stream_memory_benchmark.json`
|
||||
- `group_chat_stream_memory_benchmark_hard.json`
|
||||
第二套更长、更刁钻的压力数据，刻意加入跨话题重叠词、自然句 episode query、
|
||||
以及更容易淹没人物事实的长聊天流，用于验证修复是否具有泛化效果。
|
||||
|
||||
推荐量化指标：
|
||||
|
||||
- `search.accuracy_at_1`
|
||||
- `search.recall_at_5`
|
||||
- `search.keyword_recall_at_5`
|
||||
- `knowledge_fetcher.success_rate`
|
||||
- `profile.success_rate`
|
||||
- `writeback.success_rate`
|
||||
- `episode_generation.success_rate`
|
||||
- `negative_control.zero_hit_rate`
|
||||
|
||||
当前 fixture 结构：
|
||||
|
||||
- `simulated_stream_batches`
|
||||
用于模拟话题级聊天窗口，适合检索、episode、画像、写回等离线量化评估。
|
||||
- `runtime_trigger_streams`
|
||||
用于模拟真正能触发当前 summarizer 阈值的原生聊天流。
|
||||
这部分数据满足 `20 条 + 8 小时` 的时间触发条件，可直接用于验证
|
||||
“是否进入话题检查”与“无 Bot 发言是否被丢弃”。
|
||||
- `chat_history_records`
|
||||
用于模拟宿主将群聊话题总结后写入长期记忆的主路径。
|
||||
- `person_writebacks`
|
||||
用于模拟发送回复后的稳定人物事实写回。
|
||||
- `search_cases / time_cases / episode_cases / knowledge_fetcher_cases / profile_cases`
|
||||
用于直接驱动量化检索评估。
|
||||
- `negative_control_cases`
|
||||
用于验证“无 Bot 发言的群聊片段应被忽略”。
|
||||
|
||||
覆盖主题：
|
||||
|
||||
- 值班柜第二层的备用物资与物资报备
|
||||
- 停电夜投影仪抢救与应急灯 / 橙色延长线盘
|
||||
- 风铃观测前的温湿度计校准与无糖姜茶
|
||||
- 东侧窗边狸花猫、绿色硬壳笔记本与黄铜回形针
|
||||
- 无 Bot 参与的零食闲聊负样本
|
||||
|
||||
使用建议：
|
||||
|
||||
- 如果要验证“当前 summarizer 是否真的会被触发”，优先喂 `runtime_trigger_streams`。
|
||||
- 如果要验证“当前实现是否真正符合总结后写入和检索设计”，优先喂 `simulated_stream_batches` 与 `chat_history_records`。
|
||||
- 如果要快速跑检索、画像、episode、写回指标，直接使用 `chat_history_records + person_writebacks + cases`。
|
||||
- 如果要切换到第二套压力数据，可在运行 benchmark 前设置
|
||||
`A_MEMORIX_BENCHMARK_DATA_FILE=pytests/A_memorix_test/data/benchmarks/group_chat_stream_memory_benchmark_hard.json`。
|
||||
@@ -0,0 +1,728 @@
|
||||
{
|
||||
"meta": {
|
||||
"scenario_id": "group_chat_stream_memory_benchmark",
|
||||
"description": "面向群聊聊天流的长期记忆量化评估数据集，覆盖 Bot 参与门槛、话题收束、检索增强、人物事实写回与负样本忽略。",
|
||||
"designed_for": [
|
||||
"group_chat",
|
||||
"topic_batching",
|
||||
"bot_participation_gate",
|
||||
"runtime_trigger",
|
||||
"search_mode",
|
||||
"time_mode",
|
||||
"episode_mode",
|
||||
"aggregate_mode",
|
||||
"person_fact_writeback",
|
||||
"negative_control"
|
||||
],
|
||||
"quantitative_targets": {
|
||||
"search": {
|
||||
"accuracy_at_1": 0.4,
|
||||
"recall_at_5": 0.75,
|
||||
"keyword_recall_at_5": 0.85
|
||||
},
|
||||
"knowledge_fetcher": {
|
||||
"success_rate": 0.75,
|
||||
"keyword_recall": 0.8
|
||||
},
|
||||
"profile": {
|
||||
"success_rate": 0.67,
|
||||
"evidence_rate": 1.0
|
||||
},
|
||||
"writeback": {
|
||||
"success_rate": 0.67,
|
||||
"keyword_recall": 0.8
|
||||
},
|
||||
"episode": {
|
||||
"success_rate": 0.75,
|
||||
"keyword_recall": 0.8
|
||||
},
|
||||
"negative_control": {
|
||||
"zero_hit_rate": 1.0
|
||||
},
|
||||
"runtime_trigger": {
|
||||
"positive_trigger_rate": 1.0,
|
||||
"negative_discard_rate": 1.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"session": {
|
||||
"session_id": "qq_group_424242",
|
||||
"platform": "qq",
|
||||
"user_id": "10086",
|
||||
"group_id": "424242",
|
||||
"display_name": "松烟阁夜谈群"
|
||||
},
|
||||
"simulated_stream_batches": [
|
||||
{
|
||||
"batch_id": "supply_round_march_03",
|
||||
"topic": "值班柜第二层物资补位",
|
||||
"bot_participated": true,
|
||||
"expected_behavior": "should_be_summarized",
|
||||
"message_count": 12,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"林澈",
|
||||
"周枝",
|
||||
"秦昭",
|
||||
"唐未"
|
||||
],
|
||||
"start_time": 1772526180.0,
|
||||
"end_time": 1772526960.0,
|
||||
"messages": [
|
||||
"[2026-03-03 19:03] 林澈：空调又直吹值班桌，我先把风扇拨回二档。值班柜第二层是不是只剩一把备用钥匙了？我记得黑柄折叠伞和创可贴本来都在那层。",
|
||||
"[2026-03-03 19:04] 周枝：昨天雨太大，我把黑柄折叠伞借给来送样本的人了，回来的时候顺手挂回柜里，但创可贴我没核对。",
|
||||
"[2026-03-03 19:05] 秦昭：柜门里侧还夹着旧清单，最底下一行写着“备用钥匙、折叠伞、创可贴、充电头”，不过充电头早就改放一层了。",
|
||||
"[2026-03-03 19:06] Mai：那我记一下，第二层以后固定只放备用钥匙、黑柄折叠伞和创可贴，谁临时拿走就在群里报备，不然下次值夜的人会抓瞎。",
|
||||
"[2026-03-03 19:08] 唐未：我刚看了一眼，创可贴只剩两片，外包装已经翘边了，最好连同备用纱布一起补一盒新的。",
|
||||
"[2026-03-03 19:10] 林澈：我晚上值夜的时候最怕找东西翻半天，尤其空调风一吹头就疼，所以物资位置还是固定最省心。",
|
||||
"[2026-03-03 19:11] 周枝：我待会儿把创可贴和纱布一起补上，再把备用钥匙套个红绳，不然黑灯找起来太慢。",
|
||||
"[2026-03-03 19:12] 秦昭：折叠伞我建议继续放第二层最左边，别再和记录本挤一起，上回纸边都被伞骨刮卷了。",
|
||||
"[2026-03-03 19:14] Mai：行，我把“第二层左侧放折叠伞，中间放备用钥匙，右边放创可贴”记到今晚值班备注里。",
|
||||
"[2026-03-03 19:15] 唐未：顺便提醒一下，如果谁把第二层的物资借出去，至少在群里发一句“已借出+归还时间”，别只口头说。",
|
||||
"[2026-03-03 19:16] 周枝：收到，我今晚补完会拍一张柜内照片，省得之后又靠记忆猜。",
|
||||
"[2026-03-03 19:16] 林澈：谢谢，等柜里补齐了我就把桂花乌龙和个人杯子继续放回上层，省得和公用物资混在一起。"
|
||||
],
|
||||
"combined_text": "[2026-03-03 19:03] 林澈：空调又直吹值班桌，我先把风扇拨回二档。值班柜第二层是不是只剩一把备用钥匙了？我记得黑柄折叠伞和创可贴本来都在那层。\n[2026-03-03 19:04] 周枝：昨天雨太大，我把黑柄折叠伞借给来送样本的人了，回来的时候顺手挂回柜里，但创可贴我没核对。\n[2026-03-03 19:05] 秦昭：柜门里侧还夹着旧清单，最底下一行写着“备用钥匙、折叠伞、创可贴、充电头”，不过充电头早就改放一层了。\n[2026-03-03 19:06] Mai：那我记一下，第二层以后固定只放备用钥匙、黑柄折叠伞和创可贴，谁临时拿走就在群里报备，不然下次值夜的人会抓瞎。\n[2026-03-03 19:08] 唐未：我刚看了一眼，创可贴只剩两片，外包装已经翘边了，最好连同备用纱布一起补一盒新的。\n[2026-03-03 19:10] 林澈：我晚上值夜的时候最怕找东西翻半天，尤其空调风一吹头就疼，所以物资位置还是固定最省心。\n[2026-03-03 19:11] 周枝：我待会儿把创可贴和纱布一起补上，再把备用钥匙套个红绳，不然黑灯找起来太慢。\n[2026-03-03 19:12] 秦昭：折叠伞我建议继续放第二层最左边，别再和记录本挤一起，上回纸边都被伞骨刮卷了。\n[2026-03-03 19:14] Mai：行，我把“第二层左侧放折叠伞，中间放备用钥匙，右边放创可贴”记到今晚值班备注里。\n[2026-03-03 19:15] 唐未：顺便提醒一下，如果谁把第二层的物资借出去，至少在群里发一句“已借出+归还时间”，别只口头说。\n[2026-03-03 19:16] 周枝：收到，我今晚补完会拍一张柜内照片，省得之后又靠记忆猜。\n[2026-03-03 19:16] 林澈：谢谢，等柜里补齐了我就把桂花乌龙和个人杯子继续放回上层，省得和公用物资混在一起。",
|
||||
"expected_memory_targets": [
|
||||
"值班柜第二层",
|
||||
"备用钥匙",
|
||||
"黑柄折叠伞",
|
||||
"创可贴",
|
||||
"报备"
|
||||
]
|
||||
},
|
||||
{
|
||||
"batch_id": "blackout_projection_march_06",
|
||||
"topic": "停电夜投影仪抢救",
|
||||
"bot_participated": true,
|
||||
"expected_behavior": "should_be_summarized",
|
||||
"message_count": 11,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"秦昭",
|
||||
"唐未",
|
||||
"林澈",
|
||||
"周枝"
|
||||
],
|
||||
"start_time": 1772791980.0,
|
||||
"end_time": 1772792820.0,
|
||||
"messages": [
|
||||
"[2026-03-06 21:33] 秦昭：北墙射灯刚关，投影仪才开到一半就跳闸了，整间资料室一下子全黑，镜头还没盖上。",
|
||||
"[2026-03-06 21:34] 唐未：别先碰镜头，我去摸仪器桌下层的应急灯，上次检修后应该还塞在最里边。",
|
||||
"[2026-03-06 21:35] 林澈：我在门边摸到橙色延长线盘了，先别急着全拉开，确认是不是排插过载再说。",
|
||||
"[2026-03-06 21:36] Mai：先按顺序来，唐未开应急灯，秦昭别挪投影仪，林澈看一下是不是延长线和热风枪同时挂在同一路上。",
|
||||
"[2026-03-06 21:38] 唐未：应急灯找到了，在仪器桌下层右手边的蓝色档案盒后面，亮度够，至少能先护住镜头和电源键。",
|
||||
"[2026-03-06 21:39] 秦昭：问题找到了，热风枪和投影仪都接在橙色延长线盘上，刚才我又把扫描灯也插进去了，估计就是这个组合把闸打掉了。",
|
||||
"[2026-03-06 21:40] 周枝：我把热风枪先拔掉，扫描灯改到东墙独立插口，橙色延长线盘只留投影仪和笔记本电源。",
|
||||
"[2026-03-06 21:41] Mai：好，先恢复最少设备，镜头盖等重新上电稳定后再扣，别在黑里来回碰。",
|
||||
"[2026-03-06 21:43] 林澈：现在电回来了，投影仪风扇声正常，橙色延长线盘没有再冒热，看来就是同路负载堆太多。",
|
||||
"[2026-03-06 21:44] 秦昭：记一下，应急灯平时别乱挪，必须固定放在仪器桌下层，橙色延长线盘也只给投影相关设备用。",
|
||||
"[2026-03-06 21:47] Mai：我已经记到故障备忘里了：停电夜先找仪器桌下层应急灯，再检查橙色延长线盘负载，不要让热风枪和投影仪挂同一路。"
|
||||
],
|
||||
"combined_text": "[2026-03-06 21:33] 秦昭：北墙射灯刚关，投影仪才开到一半就跳闸了，整间资料室一下子全黑，镜头还没盖上。\n[2026-03-06 21:34] 唐未：别先碰镜头，我去摸仪器桌下层的应急灯，上次检修后应该还塞在最里边。\n[2026-03-06 21:35] 林澈：我在门边摸到橙色延长线盘了，先别急着全拉开，确认是不是排插过载再说。\n[2026-03-06 21:36] Mai：先按顺序来，唐未开应急灯，秦昭别挪投影仪，林澈看一下是不是延长线和热风枪同时挂在同一路上。\n[2026-03-06 21:38] 唐未：应急灯找到了，在仪器桌下层右手边的蓝色档案盒后面，亮度够，至少能先护住镜头和电源键。\n[2026-03-06 21:39] 秦昭：问题找到了，热风枪和投影仪都接在橙色延长线盘上，刚才我又把扫描灯也插进去了，估计就是这个组合把闸打掉了。\n[2026-03-06 21:40] 周枝：我把热风枪先拔掉，扫描灯改到东墙独立插口，橙色延长线盘只留投影仪和笔记本电源。\n[2026-03-06 21:41] Mai：好，先恢复最少设备，镜头盖等重新上电稳定后再扣，别在黑里来回碰。\n[2026-03-06 21:43] 林澈：现在电回来了，投影仪风扇声正常，橙色延长线盘没有再冒热，看来就是同路负载堆太多。\n[2026-03-06 21:44] 秦昭：记一下，应急灯平时别乱挪，必须固定放在仪器桌下层，橙色延长线盘也只给投影相关设备用。\n[2026-03-06 21:47] Mai：我已经记到故障备忘里了：停电夜先找仪器桌下层应急灯，再检查橙色延长线盘负载，不要让热风枪和投影仪挂同一路。",
|
||||
"expected_memory_targets": [
|
||||
"停电夜",
|
||||
"投影仪",
|
||||
"应急灯",
|
||||
"橙色延长线盘",
|
||||
"仪器桌下层"
|
||||
]
|
||||
},
|
||||
{
|
||||
"batch_id": "wind_bell_observation_march_10",
|
||||
"topic": "风铃观测与姜茶准备",
|
||||
"bot_participated": true,
|
||||
"expected_behavior": "should_be_summarized",
|
||||
"message_count": 12,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"唐未",
|
||||
"林澈",
|
||||
"周枝",
|
||||
"许棠"
|
||||
],
|
||||
"start_time": 1773136020.0,
|
||||
"end_time": 1773136980.0,
|
||||
"messages": [
|
||||
"[2026-03-10 20:27] 唐未：周六夜里要做风铃塔观测，我下午先去校准温湿度计，镜片也顺便擦一遍，不然露点记录会飘。",
|
||||
"[2026-03-10 20:28] 林澈：我可以把南平台的小风扇带上去，但别让我坐空调口边上值守，上次吹得我偏头痛一整晚。",
|
||||
"[2026-03-10 20:30] 周枝：观测箱里还剩两包姜片，我去值班室再补一盒无糖姜茶，甜的那种唐未不喝。",
|
||||
"[2026-03-10 20:31] 许棠：屋顶东侧栏杆边那只风铃这两天响得特别密，最好把记录本提前夹在硬板夹上，别到时一手按纸一手扶灯。",
|
||||
"[2026-03-10 20:32] Mai：那就按这个分工：唐未负责温湿度计和镜片，周枝准备无糖姜茶和记录夹，林澈盯现场风向和风扇位置。",
|
||||
"[2026-03-10 20:35] 唐未：我会把校准后的时间写在第一页右上角，免得后面整理的时候又分不清哪组数据是校准前的。",
|
||||
"[2026-03-10 20:37] 林澈：如果夜里温差太大，我会把风扇固定在二档，既能带走雾气，又不至于把纸吹跑。",
|
||||
"[2026-03-10 20:39] 周枝：姜茶我放保温壶里，标签写“无糖”，免得有人顺手加糖包，唐未每次都喝不下去。",
|
||||
"[2026-03-10 20:41] 许棠：记录本别用软封皮，上回屋顶起风，边角被栏杆刮得卷起来，还是硬板夹最稳。",
|
||||
"[2026-03-10 20:43] Mai：再补一条，风铃塔观测前先在群里报“温湿度计已校准”，这样后面谁接手都知道状态。",
|
||||
"[2026-03-10 20:45] 唐未：收到，我会先发校准完成，再把镜片状态和气温一起报出来。",
|
||||
"[2026-03-10 20:46] 林澈：那我带桂花乌龙给自己，公用保温壶就只放无糖姜茶，别把味道混了。"
|
||||
],
|
||||
"combined_text": "[2026-03-10 20:27] 唐未：周六夜里要做风铃塔观测，我下午先去校准温湿度计，镜片也顺便擦一遍，不然露点记录会飘。\n[2026-03-10 20:28] 林澈：我可以把南平台的小风扇带上去，但别让我坐空调口边上值守，上次吹得我偏头痛一整晚。\n[2026-03-10 20:30] 周枝：观测箱里还剩两包姜片，我去值班室再补一盒无糖姜茶，甜的那种唐未不喝。\n[2026-03-10 20:31] 许棠：屋顶东侧栏杆边那只风铃这两天响得特别密，最好把记录本提前夹在硬板夹上，别到时一手按纸一手扶灯。\n[2026-03-10 20:32] Mai：那就按这个分工：唐未负责温湿度计和镜片，周枝准备无糖姜茶和记录夹，林澈盯现场风向和风扇位置。\n[2026-03-10 20:35] 唐未：我会把校准后的时间写在第一页右上角，免得后面整理的时候又分不清哪组数据是校准前的。\n[2026-03-10 20:37] 林澈：如果夜里温差太大，我会把风扇固定在二档，既能带走雾气，又不至于把纸吹跑。\n[2026-03-10 20:39] 周枝：姜茶我放保温壶里，标签写“无糖”，免得有人顺手加糖包，唐未每次都喝不下去。\n[2026-03-10 20:41] 许棠：记录本别用软封皮，上回屋顶起风，边角被栏杆刮得卷起来，还是硬板夹最稳。\n[2026-03-10 20:43] Mai：再补一条，风铃塔观测前先在群里报“温湿度计已校准”，这样后面谁接手都知道状态。\n[2026-03-10 20:45] 唐未：收到，我会先发校准完成，再把镜片状态和气温一起报出来。\n[2026-03-10 20:46] 林澈：那我带桂花乌龙给自己，公用保温壶就只放无糖姜茶，别把味道混了。",
|
||||
"expected_memory_targets": [
|
||||
"风铃塔观测",
|
||||
"温湿度计",
|
||||
"无糖姜茶",
|
||||
"硬板夹",
|
||||
"屋顶东侧栏杆"
|
||||
]
|
||||
},
|
||||
{
|
||||
"batch_id": "archive_cat_march_14",
|
||||
"topic": "东侧窗边狸花猫与绿色笔记本",
|
||||
"bot_participated": true,
|
||||
"expected_behavior": "should_be_summarized",
|
||||
"message_count": 11,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"许棠",
|
||||
"周枝",
|
||||
"秦昭",
|
||||
"林澈"
|
||||
],
|
||||
"start_time": 1773478440.0,
|
||||
"end_time": 1773479280.0,
|
||||
"messages": [
|
||||
"[2026-03-14 19:34] 许棠：东侧窗边那只狸花猫又钻进档案室了，刚才直接跳上长桌，把绿色硬壳笔记本踩得翻到中间页。",
|
||||
"[2026-03-14 19:35] 周枝：长桌上不是还夹着黄铜回形针和旧雨量页吗？猫要是蹭一下，纸页顺着窗缝就容易飞。",
|
||||
"[2026-03-14 19:36] 秦昭：我刚把窗缝先关小了，绿色硬壳笔记本已经挪到蓝色档案盒上面，黄铜回形针我也收回铁盘里了。",
|
||||
"[2026-03-14 19:38] Mai：先别赶猫，确认一下它是不是又往暖气后面钻。长桌这边只保留绿色笔记本和今晚要抄的旧雨量页，其他散物都收走。",
|
||||
"[2026-03-14 19:40] 林澈：狸花猫刚从暖气后面出来，蹭了一圈又去窗边晒了，至少没再踩记录纸。我顺手把长桌右侧的纸镇也压上了。",
|
||||
"[2026-03-14 19:41] 许棠：绿色笔记本第七页记的是去年秋天的风速补注，别被猫爪勾破了，那页我明天还要录系统。",
|
||||
"[2026-03-14 19:43] 周枝：以后东侧窗边长桌如果要摊资料，先把黄铜回形针和纸镇放好，不然有风再加猫，纸真的收不回来。",
|
||||
"[2026-03-14 19:45] 秦昭：蓝色档案盒我也顺手换到桌角了，避免猫跳上来时把整盒推下去。",
|
||||
"[2026-03-14 19:46] Mai：我记个规则：东侧窗边长桌只放绿色硬壳笔记本、当次要抄的纸页和纸镇；黄铜回形针统一回铁盘。",
|
||||
"[2026-03-14 19:48] 林澈：那只狸花猫现在缩在窗台垫子上了，看样子只是找暖和，不是故意拆台。",
|
||||
"[2026-03-14 19:48] 许棠：收到，明天我会先去看第七页，再决定要不要把整本绿色笔记本转移到里间。"
|
||||
],
|
||||
"combined_text": "[2026-03-14 19:34] 许棠：东侧窗边那只狸花猫又钻进档案室了，刚才直接跳上长桌，把绿色硬壳笔记本踩得翻到中间页。\n[2026-03-14 19:35] 周枝：长桌上不是还夹着黄铜回形针和旧雨量页吗？猫要是蹭一下，纸页顺着窗缝就容易飞。\n[2026-03-14 19:36] 秦昭：我刚把窗缝先关小了，绿色硬壳笔记本已经挪到蓝色档案盒上面，黄铜回形针我也收回铁盘里了。\n[2026-03-14 19:38] Mai：先别赶猫，确认一下它是不是又往暖气后面钻。长桌这边只保留绿色笔记本和今晚要抄的旧雨量页，其他散物都收走。\n[2026-03-14 19:40] 林澈：狸花猫刚从暖气后面出来，蹭了一圈又去窗边晒了，至少没再踩记录纸。我顺手把长桌右侧的纸镇也压上了。\n[2026-03-14 19:41] 许棠：绿色笔记本第七页记的是去年秋天的风速补注，别被猫爪勾破了，那页我明天还要录系统。\n[2026-03-14 19:43] 周枝：以后东侧窗边长桌如果要摊资料，先把黄铜回形针和纸镇放好，不然有风再加猫，纸真的收不回来。\n[2026-03-14 19:45] 秦昭：蓝色档案盒我也顺手换到桌角了，避免猫跳上来时把整盒推下去。\n[2026-03-14 19:46] Mai：我记个规则：东侧窗边长桌只放绿色硬壳笔记本、当次要抄的纸页和纸镇；黄铜回形针统一回铁盘。\n[2026-03-14 19:48] 林澈：那只狸花猫现在缩在窗台垫子上了，看样子只是找暖和，不是故意拆台。\n[2026-03-14 19:48] 许棠：收到，明天我会先去看第七页，再决定要不要把整本绿色笔记本转移到里间。",
|
||||
"expected_memory_targets": [
|
||||
"东侧窗边",
|
||||
"狸花猫",
|
||||
"绿色硬壳笔记本",
|
||||
"黄铜回形针",
|
||||
"蓝色档案盒"
|
||||
]
|
||||
},
|
||||
{
|
||||
"batch_id": "snack_gossip_march_15_negative",
|
||||
"topic": "无 Bot 参与的零食闲聊负样本",
|
||||
"bot_participated": false,
|
||||
"expected_behavior": "ignored_by_summarizer_without_bot_message",
|
||||
"message_count": 8,
|
||||
"participants": [
|
||||
"许棠",
|
||||
"周枝",
|
||||
"秦昭",
|
||||
"林澈"
|
||||
],
|
||||
"start_time": 1773565320.0,
|
||||
"end_time": 1773566160.0,
|
||||
"messages": [
|
||||
"[2026-03-15 19:42] 许棠：我准备给周末值班买海盐柠檬饼干，你们有人忌口吗？",
|
||||
"[2026-03-15 19:43] 周枝：我不吃太甜的，但海盐的可以，顺便来点原味苏打更稳。",
|
||||
"[2026-03-15 19:45] 秦昭：我想要辣味海苔片，饼干别买太碎的，上次全压成粉了。",
|
||||
"[2026-03-15 19:46] 林澈：如果有无糖薄荷糖也帮我带一盒，值夜后半段嘴里太淡容易犯困。",
|
||||
"[2026-03-15 19:48] 许棠：那我就下单海盐柠檬饼干、原味苏打、辣味海苔和无糖薄荷糖。",
|
||||
"[2026-03-15 19:50] 周枝：别忘了备注送到北门值班室，不然又会被前台放到快递架最里面。",
|
||||
"[2026-03-15 19:54] 秦昭：海苔别买大片装，碎屑掉在键盘里太难清。",
|
||||
"[2026-03-15 19:56] 林澈：收到，等到了我去北门拿。"
|
||||
],
|
||||
"combined_text": "[2026-03-15 19:42] 许棠：我准备给周末值班买海盐柠檬饼干，你们有人忌口吗？\n[2026-03-15 19:43] 周枝：我不吃太甜的，但海盐的可以，顺便来点原味苏打更稳。\n[2026-03-15 19:45] 秦昭：我想要辣味海苔片，饼干别买太碎的，上次全压成粉了。\n[2026-03-15 19:46] 林澈：如果有无糖薄荷糖也帮我带一盒，值夜后半段嘴里太淡容易犯困。\n[2026-03-15 19:48] 许棠：那我就下单海盐柠檬饼干、原味苏打、辣味海苔和无糖薄荷糖。\n[2026-03-15 19:50] 周枝：别忘了备注送到北门值班室，不然又会被前台放到快递架最里面。\n[2026-03-15 19:54] 秦昭：海苔别买大片装，碎屑掉在键盘里太难清。\n[2026-03-15 19:56] 林澈：收到，等到了我去北门拿。",
|
||||
"expected_memory_targets": [
|
||||
"海盐柠檬饼干",
|
||||
"原味苏打",
|
||||
"辣味海苔",
|
||||
"无糖薄荷糖"
|
||||
]
|
||||
}
|
||||
],
|
||||
"runtime_trigger_streams": [
|
||||
{
|
||||
"stream_id": "runtime_supply_trigger_march_18",
|
||||
"topic": "值班柜第二层物资规则长流触发样本",
|
||||
"trigger_mode": "time_threshold",
|
||||
"elapsed_since_last_check_hours": 8.7,
|
||||
"bot_participated": true,
|
||||
"expected_check_outcome": "should_trigger_topic_check_and_pass_bot_gate",
|
||||
"expected_next_stage": "topic_cache_should_update",
|
||||
"message_count": 22,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"林澈",
|
||||
"周枝",
|
||||
"秦昭",
|
||||
"唐未"
|
||||
],
|
||||
"start_time": 1773795720.0,
|
||||
"end_time": 1773827160.0,
|
||||
"messages": [
|
||||
"[2026-03-18 09:02] 林澈：我早上开柜门的时候又被空调风正面吹到，顺手看了一眼，值班柜第二层现在只有备用钥匙和半包创可贴了。",
|
||||
"[2026-03-18 09:05] 周枝：黑柄折叠伞昨晚借给送样的人还没放回，我记得归还口头说的是今天中午前。",
|
||||
"[2026-03-18 09:07] 秦昭：柜门里那张旧清单我还没撕，最下面还是写着备用钥匙、折叠伞、创可贴，不过字已经糊了。",
|
||||
"[2026-03-18 09:09] 唐未：创可贴我刚翻过，只剩一条完整包装，纱布也只余两片，今天最好一起补齐。",
|
||||
"[2026-03-18 09:12] Mai：先别各自乱挪，今天这轮就按第二层只放备用钥匙、黑柄折叠伞、创可贴和备用纱布来整理，借出统一在群里报备。",
|
||||
"[2026-03-18 09:16] 林澈：收到，我主要担心晚上值夜的时候黑灯摸东西太慢，所以备用钥匙最好再挂个醒目的红绳。",
|
||||
"[2026-03-18 09:24] 周枝：我中午前会带红绳过去，顺手把创可贴和纱布一起补上，再拍一张第二层的照片。",
|
||||
"[2026-03-18 09:31] 秦昭：折叠伞还是建议固定在第二层最左边，钥匙放中间，创可贴和纱布靠右，别再和记录本混在一起。",
|
||||
"[2026-03-18 10:02] 唐未：我去医务柜那边借到了一盒新创可贴和一包小纱布，午休回来就补进去。",
|
||||
"[2026-03-18 10:28] Mai：补充一条，凡是从第二层拿走物资，都要发“已借出+用途+预计归还时间”，不要只在现场口头说。",
|
||||
"[2026-03-18 11:14] 周枝：我先把备用钥匙套上红绳了，照片也拍了，不过黑柄折叠伞还在门口晾水，等干一点再放回柜里。",
|
||||
"[2026-03-18 12:43] 林澈：我刚在值班桌后面找到那把黑柄折叠伞，伞骨没问题，就是伞套湿着，我先挂在柜侧透气。",
|
||||
"[2026-03-18 13:26] 秦昭：旧清单我重新写了一份，第二层现在明确写成“左伞中钥匙右医用小物”，这样新来的人也看得懂。",
|
||||
"[2026-03-18 14:18] 唐未：新的创可贴和纱布都补齐了，我另外塞了一卷医用胶带，但没和创可贴压在一起，怕拿的时候全带出来。",
|
||||
"[2026-03-18 15:07] Mai：医用胶带别挤占创可贴的位置，优先保证备用钥匙、黑柄折叠伞、创可贴、纱布这四样一眼能看到。",
|
||||
"[2026-03-18 15:52] 周枝：现在第二层左边是折叠伞，中间是带红绳的备用钥匙，右边是创可贴和纱布，拍照存档已经发群文件了。",
|
||||
"[2026-03-18 16:21] 林澈：这样就清楚多了，我自己的桂花乌龙和杯子还是继续放上层，不和公用物资混放。",
|
||||
"[2026-03-18 16:57] 秦昭：我把旧清单撕掉了，柜门里只留新版规则，写了借出后必须报备和归还时间。",
|
||||
"[2026-03-18 17:08] 唐未：医用小物这边都齐了，晚上如果还有人借伞或拿创可贴，记得照新版格式报一下。",
|
||||
"[2026-03-18 17:24] Mai：今晚值班备注我已经更新成固定摆放规则了，之后默认按这版执行，除非群里另行通知。",
|
||||
"[2026-03-18 17:38] 周枝：我补一条，借出的东西最好二十四小时内归还，超过时间就在群里继续报状态。",
|
||||
"[2026-03-18 17:46] 林澈：这个规则对夜班太友好了，至少以后不会一边被空调吹一边在柜里翻半天。"
|
||||
],
|
||||
"expected_memory_targets": [
|
||||
"值班柜第二层",
|
||||
"备用钥匙",
|
||||
"黑柄折叠伞",
|
||||
"创可贴",
|
||||
"纱布",
|
||||
"报备"
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream_id": "runtime_snack_gossip_trigger_march_22_negative",
|
||||
"topic": "无 Bot 参与的零食拼单长流负样本",
|
||||
"trigger_mode": "time_threshold",
|
||||
"elapsed_since_last_check_hours": 8.6,
|
||||
"bot_participated": false,
|
||||
"expected_check_outcome": "should_trigger_topic_check_but_be_discarded_without_bot_message",
|
||||
"expected_next_stage": "topic_cache_should_not_update",
|
||||
"message_count": 21,
|
||||
"participants": [
|
||||
"许棠",
|
||||
"周枝",
|
||||
"秦昭",
|
||||
"林澈"
|
||||
],
|
||||
"start_time": 1774144860.0,
|
||||
"end_time": 1774175700.0,
|
||||
"messages": [
|
||||
"[2026-03-22 10:01] 许棠：周末值班太长了，我准备拼一单零食，先问下大家海盐柠檬饼干和原味苏打能不能接受。",
|
||||
"[2026-03-22 10:12] 周枝：原味苏打可以，海盐柠檬也行，但别买太甜的夹心款，我夜里吃那个会腻。",
|
||||
"[2026-03-22 10:20] 秦昭：我想要辣味海苔片，别选大片易掉渣的版本，上回碎屑全进键盘缝了。",
|
||||
"[2026-03-22 10:33] 林澈：如果有无糖薄荷糖也帮我带一盒，后半夜嘴里太淡的时候容易犯困。",
|
||||
"[2026-03-22 10:52] 许棠：那我先记海盐柠檬饼干、原味苏打、辣味海苔和无糖薄荷糖，坚果你们有人想加吗？",
|
||||
"[2026-03-22 11:15] 周枝：坚果少一点吧，值班桌一忙起来很容易忘记封口，还是独立小包装更稳。",
|
||||
"[2026-03-22 11:42] 秦昭：海苔一定别买油太大的，拿文件之前还得擦手太麻烦。",
|
||||
"[2026-03-22 12:08] 林澈：如果可以的话顺便来两包原味小麻花，脆一点但别掉很多渣。",
|
||||
"[2026-03-22 12:44] 许棠：行，我把小麻花也记上，不过总价快到免配送门槛了，再凑一点就够。",
|
||||
"[2026-03-22 13:17] 周枝：可以补几包茶包，别太香精味重，清淡一点的红茶或者大麦茶都行。",
|
||||
"[2026-03-22 13:58] 秦昭：辣条就别买了，油会蹭到键帽上，还是海苔和苏打这种最安全。",
|
||||
"[2026-03-22 14:26] 林澈：如果店里有小盒装薄荷糖，比大袋装方便，值夜带着走也不会散一桌。",
|
||||
"[2026-03-22 15:09] 许棠：配送时间我打算约到晚上六点半左右，正好你们晚饭后能一起收。",
|
||||
"[2026-03-22 15:47] 周枝：每个人单独贴名字吧，不然海盐柠檬和原味苏打长得太像，容易拿混。",
|
||||
"[2026-03-22 16:13] 秦昭：再加一包湿巾也行，吃完零食直接擦手，省得摸鼠标留油印。",
|
||||
"[2026-03-22 16:42] 林澈：奶味太重的糖就不要了，我夜里喝水本来就少，吃那个更容易口干。",
|
||||
"[2026-03-22 17:01] 许棠：配送地址我本来想填前台，后来还是觉得北门值班室最稳，省得被塞到最里面。",
|
||||
"[2026-03-22 17:18] 周枝：对，直接送北门值班室，备注写清楚到门口电话联系，不然外卖员总找不到。",
|
||||
"[2026-03-22 17:46] 秦昭：如果凑单还差一点，就补最普通的瓜子，别买调味太重的版本。",
|
||||
"[2026-03-22 18:12] 林澈：到了之后我去门口拿，顺便把无糖薄荷糖先分出来，免得被当成普通糖。",
|
||||
"[2026-03-22 18:35] 许棠：已经下单了，截图我发群里，内容就是海盐柠檬饼干、原味苏打、辣味海苔、无糖薄荷糖、小麻花和茶包。"
|
||||
],
|
||||
"expected_memory_targets": []
|
||||
}
|
||||
],
|
||||
"import_payload": {
|
||||
"paragraphs": [
|
||||
{
|
||||
"content": "东侧档案室窗边长桌通常用于整理当晚资料，绿色硬壳笔记本、黄铜回形针和纸镇经常在这里配套出现。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark",
|
||||
"knowledge_type": "narrative",
|
||||
"entities": [
|
||||
"东侧档案室",
|
||||
"长桌",
|
||||
"绿色硬壳笔记本",
|
||||
"黄铜回形针",
|
||||
"纸镇"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "停电或抢修投影设备时，仪器桌下层的应急灯和橙色延长线盘是最常被提到的两个关键物品。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"应急灯",
|
||||
"橙色延长线盘",
|
||||
"仪器桌下层",
|
||||
"投影设备"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "周枝平时看管值班柜第二层的备用钥匙、黑柄折叠伞和创可贴，借出时要求在群里报备。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"周枝",
|
||||
"值班柜第二层",
|
||||
"备用钥匙",
|
||||
"黑柄折叠伞",
|
||||
"创可贴"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "唐未在观测前通常先校准温湿度计和擦镜片，并且只喝无糖姜茶。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"唐未",
|
||||
"温湿度计",
|
||||
"镜片",
|
||||
"无糖姜茶"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "林澈值夜时不喜欢空调直吹，常把风扇固定在二档，同时会自带桂花乌龙。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"林澈",
|
||||
"空调直吹",
|
||||
"风扇二档",
|
||||
"桂花乌龙"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "秦昭做投影演示前会先关北墙射灯，避免镜头反光，再检查投影设备是否与大功率工具共用同一路电源。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"秦昭",
|
||||
"北墙射灯",
|
||||
"镜头反光",
|
||||
"投影设备"
|
||||
]
|
||||
}
|
||||
],
|
||||
"relations": [
|
||||
{
|
||||
"subject": "周枝",
|
||||
"predicate": "看管",
|
||||
"object": "值班柜第二层"
|
||||
},
|
||||
{
|
||||
"subject": "值班柜第二层",
|
||||
"predicate": "存放",
|
||||
"object": "黑柄折叠伞"
|
||||
},
|
||||
{
|
||||
"subject": "唐未",
|
||||
"predicate": "负责",
|
||||
"object": "温湿度计校准"
|
||||
},
|
||||
{
|
||||
"subject": "唐未",
|
||||
"predicate": "偏好",
|
||||
"object": "无糖姜茶"
|
||||
},
|
||||
{
|
||||
"subject": "林澈",
|
||||
"predicate": "不喜欢",
|
||||
"object": "空调直吹"
|
||||
},
|
||||
{
|
||||
"subject": "秦昭",
|
||||
"predicate": "会先关闭",
|
||||
"object": "北墙射灯"
|
||||
},
|
||||
{
|
||||
"subject": "东侧档案室长桌",
|
||||
"predicate": "摆放",
|
||||
"object": "绿色硬壳笔记本"
|
||||
}
|
||||
]
|
||||
},
|
||||
"chat_history_records": [
|
||||
{
|
||||
"record_id": 920001,
|
||||
"theme": "值班柜第二层物资补位",
|
||||
"summary": "群里确认值班柜第二层固定放备用钥匙、黑柄折叠伞和创可贴，借出后必须在群里报备；林澈再次提到自己值夜时怕空调直吹，会把风扇调到二档。",
|
||||
"participants": [
|
||||
"Mai",
|
||||
"林澈",
|
||||
"周枝",
|
||||
"秦昭",
|
||||
"唐未"
|
||||
],
|
||||
"start_time": 1772526180.0,
|
||||
"end_time": 1772526960.0,
|
||||
"original_text": "[2026-03-03 19:03] 林澈：空调又直吹值班桌，我先把风扇拨回二档。值班柜第二层是不是只剩一把备用钥匙了？我记得黑柄折叠伞和创可贴本来都在那层。\n[2026-03-03 19:04] 周枝：昨天雨太大，我把黑柄折叠伞借给来送样本的人了，回来的时候顺手挂回柜里，但创可贴我没核对。\n[2026-03-03 19:05] 秦昭：柜门里侧还夹着旧清单，最底下一行写着“备用钥匙、折叠伞、创可贴、充电头”，不过充电头早就改放一层了。\n[2026-03-03 19:06] Mai：那我记一下，第二层以后固定只放备用钥匙、黑柄折叠伞和创可贴，谁临时拿走就在群里报备，不然下次值夜的人会抓瞎。\n[2026-03-03 19:08] 唐未：我刚看了一眼，创可贴只剩两片，外包装已经翘边了，最好连同备用纱布一起补一盒新的。\n[2026-03-03 19:10] 林澈：我晚上值夜的时候最怕找东西翻半天，尤其空调风一吹头就疼，所以物资位置还是固定最省心。\n[2026-03-03 19:11] 周枝：我待会儿把创可贴和纱布一起补上，再把备用钥匙套个红绳，不然黑灯找起来太慢。\n[2026-03-03 19:12] 秦昭：折叠伞我建议继续放第二层最左边，别再和记录本挤一起，上回纸边都被伞骨刮卷了。\n[2026-03-03 19:14] Mai：行，我把“第二层左侧放折叠伞，中间放备用钥匙，右边放创可贴”记到今晚值班备注里。\n[2026-03-03 19:15] 唐未：顺便提醒一下，如果谁把第二层的物资借出去，至少在群里发一句“已借出+归还时间”，别只口头说。\n[2026-03-03 19:16] 周枝：收到，我今晚补完会拍一张柜内照片，省得之后又靠记忆猜。\n[2026-03-03 19:16] 林澈：谢谢，等柜里补齐了我就把桂花乌龙和个人杯子继续放回上层，省得和公用物资混在一起。"
|
||||
},
|
||||
{
|
||||
"record_id": 920002,
|
||||
"theme": "停电夜投影仪抢救",
|
||||
"summary": "资料室停电后，群里确认应急灯固定放在仪器桌下层，橙色延长线盘只给投影相关设备使用；问题由热风枪、扫描灯和投影仪共路导致。",
|
||||
"participants": [
|
||||
"Mai",
|
||||
"秦昭",
|
||||
"唐未",
|
||||
"林澈",
|
||||
"周枝"
|
||||
],
|
||||
"start_time": 1772791980.0,
|
||||
"end_time": 1772792820.0,
|
||||
"original_text": "[2026-03-06 21:33] 秦昭：北墙射灯刚关，投影仪才开到一半就跳闸了，整间资料室一下子全黑，镜头还没盖上。\n[2026-03-06 21:34] 唐未：别先碰镜头，我去摸仪器桌下层的应急灯，上次检修后应该还塞在最里边。\n[2026-03-06 21:35] 林澈：我在门边摸到橙色延长线盘了，先别急着全拉开，确认是不是排插过载再说。\n[2026-03-06 21:36] Mai：先按顺序来，唐未开应急灯，秦昭别挪投影仪，林澈看一下是不是延长线和热风枪同时挂在同一路上。\n[2026-03-06 21:38] 唐未：应急灯找到了，在仪器桌下层右手边的蓝色档案盒后面，亮度够，至少能先护住镜头和电源键。\n[2026-03-06 21:39] 秦昭：问题找到了，热风枪和投影仪都接在橙色延长线盘上，刚才我又把扫描灯也插进去了，估计就是这个组合把闸打掉了。\n[2026-03-06 21:40] 周枝：我把热风枪先拔掉，扫描灯改到东墙独立插口，橙色延长线盘只留投影仪和笔记本电源。\n[2026-03-06 21:41] Mai：好，先恢复最少设备，镜头盖等重新上电稳定后再扣，别在黑里来回碰。\n[2026-03-06 21:43] 林澈：现在电回来了，投影仪风扇声正常，橙色延长线盘没有再冒热，看来就是同路负载堆太多。\n[2026-03-06 21:44] 秦昭：记一下，应急灯平时别乱挪，必须固定放在仪器桌下层，橙色延长线盘也只给投影相关设备用。\n[2026-03-06 21:47] Mai：我已经记到故障备忘里了：停电夜先找仪器桌下层应急灯，再检查橙色延长线盘负载，不要让热风枪和投影仪挂同一路。"
|
||||
},
|
||||
{
|
||||
"record_id": 920003,
|
||||
"theme": "风铃观测与姜茶准备",
|
||||
"summary": "群里安排周六夜间风铃塔观测，唐未负责校准温湿度计和擦镜片，周枝准备无糖姜茶与记录夹，林澈负责风向与风扇位置，观测前需先报温湿度计已校准。",
|
||||
"participants": [
|
||||
"Mai",
|
||||
"唐未",
|
||||
"林澈",
|
||||
"周枝",
|
||||
"许棠"
|
||||
],
|
||||
"start_time": 1773136020.0,
|
||||
"end_time": 1773136980.0,
|
||||
"original_text": "[2026-03-10 20:27] 唐未：周六夜里要做风铃塔观测，我下午先去校准温湿度计，镜片也顺便擦一遍，不然露点记录会飘。\n[2026-03-10 20:28] 林澈：我可以把南平台的小风扇带上去，但别让我坐空调口边上值守，上次吹得我偏头痛一整晚。\n[2026-03-10 20:30] 周枝：观测箱里还剩两包姜片，我去值班室再补一盒无糖姜茶，甜的那种唐未不喝。\n[2026-03-10 20:31] 许棠：屋顶东侧栏杆边那只风铃这两天响得特别密，最好把记录本提前夹在硬板夹上，别到时一手按纸一手扶灯。\n[2026-03-10 20:32] Mai：那就按这个分工：唐未负责温湿度计和镜片，周枝准备无糖姜茶和记录夹，林澈盯现场风向和风扇位置。\n[2026-03-10 20:35] 唐未：我会把校准后的时间写在第一页右上角，免得后面整理的时候又分不清哪组数据是校准前的。\n[2026-03-10 20:37] 林澈：如果夜里温差太大，我会把风扇固定在二档，既能带走雾气，又不至于把纸吹跑。\n[2026-03-10 20:39] 周枝：姜茶我放保温壶里，标签写“无糖”，免得有人顺手加糖包，唐未每次都喝不下去。\n[2026-03-10 20:41] 许棠：记录本别用软封皮，上回屋顶起风，边角被栏杆刮得卷起来，还是硬板夹最稳。\n[2026-03-10 20:43] Mai：再补一条，风铃塔观测前先在群里报“温湿度计已校准”，这样后面谁接手都知道状态。\n[2026-03-10 20:45] 唐未：收到，我会先发校准完成，再把镜片状态和气温一起报出来。\n[2026-03-10 20:46] 林澈：那我带桂花乌龙给自己，公用保温壶就只放无糖姜茶，别把味道混了。"
|
||||
},
|
||||
{
|
||||
"record_id": 920004,
|
||||
"theme": "东侧窗边狸花猫与绿色笔记本",
|
||||
"summary": "档案室东侧窗边的狸花猫跳上长桌，群里因此整理了绿色硬壳笔记本、黄铜回形针、蓝色档案盒和纸镇的摆放规则，避免纸页被风或猫弄乱。",
|
||||
"participants": [
|
||||
"Mai",
|
||||
"许棠",
|
||||
"周枝",
|
||||
"秦昭",
|
||||
"林澈"
|
||||
],
|
||||
"start_time": 1773478440.0,
|
||||
"end_time": 1773479280.0,
|
||||
"original_text": "[2026-03-14 19:34] 许棠：东侧窗边那只狸花猫又钻进档案室了，刚才直接跳上长桌，把绿色硬壳笔记本踩得翻到中间页。\n[2026-03-14 19:35] 周枝：长桌上不是还夹着黄铜回形针和旧雨量页吗？猫要是蹭一下，纸页顺着窗缝就容易飞。\n[2026-03-14 19:36] 秦昭：我刚把窗缝先关小了，绿色硬壳笔记本已经挪到蓝色档案盒上面，黄铜回形针我也收回铁盘里了。\n[2026-03-14 19:38] Mai：先别赶猫，确认一下它是不是又往暖气后面钻。长桌这边只保留绿色笔记本和今晚要抄的旧雨量页，其他散物都收走。\n[2026-03-14 19:40] 林澈：狸花猫刚从暖气后面出来，蹭了一圈又去窗边晒了，至少没再踩记录纸。我顺手把长桌右侧的纸镇也压上了。\n[2026-03-14 19:41] 许棠：绿色笔记本第七页记的是去年秋天的风速补注，别被猫爪勾破了，那页我明天还要录系统。\n[2026-03-14 19:43] 周枝：以后东侧窗边长桌如果要摊资料，先把黄铜回形针和纸镇放好，不然有风再加猫，纸真的收不回来。\n[2026-03-14 19:45] 秦昭：蓝色档案盒我也顺手换到桌角了，避免猫跳上来时把整盒推下去。\n[2026-03-14 19:46] Mai：我记个规则：东侧窗边长桌只放绿色硬壳笔记本、当次要抄的纸页和纸镇；黄铜回形针统一回铁盘。\n[2026-03-14 19:48] 林澈：那只狸花猫现在缩在窗台垫子上了，看样子只是找暖和，不是故意拆台。\n[2026-03-14 19:48] 许棠：收到，明天我会先去看第七页，再决定要不要把整本绿色笔记本转移到里间。"
|
||||
}
|
||||
],
|
||||
"person_writebacks": [
|
||||
{
|
||||
"person_id": "6c4a50b4f4f34bdbb1a7a1c1e56c9001",
|
||||
"person_name": "林澈",
|
||||
"memory_content": "林澈值夜时不喜欢空调直吹，通常把风扇固定在二档，并且会自带桂花乌龙。",
|
||||
"expected_keywords": [
|
||||
"林澈",
|
||||
"空调直吹",
|
||||
"风扇",
|
||||
"二档",
|
||||
"桂花乌龙"
|
||||
]
|
||||
},
|
||||
{
|
||||
"person_id": "cbfd79d4680849b5ac7e23a3f6f09002",
|
||||
"person_name": "唐未",
|
||||
"memory_content": "唐未每次观测前都会先校准温湿度计和擦镜片，而且只喝无糖姜茶。",
|
||||
"expected_keywords": [
|
||||
"唐未",
|
||||
"温湿度计",
|
||||
"校准",
|
||||
"镜片",
|
||||
"无糖姜茶"
|
||||
]
|
||||
},
|
||||
{
|
||||
"person_id": "a6eb73d41251472a8d229f1202df9003",
|
||||
"person_name": "周枝",
|
||||
"memory_content": "周枝负责补和值班柜第二层的备用钥匙、黑柄折叠伞和创可贴，借出后会在群里报备。",
|
||||
"expected_keywords": [
|
||||
"周枝",
|
||||
"值班柜第二层",
|
||||
"备用钥匙",
|
||||
"黑柄折叠伞",
|
||||
"创可贴"
|
||||
]
|
||||
}
|
||||
],
|
||||
"search_cases": [
|
||||
{
|
||||
"query": "值班柜第二层 备用钥匙 折叠伞 创可贴",
|
||||
"expected_keywords": [
|
||||
"值班柜第二层",
|
||||
"备用钥匙",
|
||||
"黑柄折叠伞",
|
||||
"创可贴"
|
||||
],
|
||||
"minimum_keyword_hits": 2
|
||||
},
|
||||
{
|
||||
"query": "停电夜 投影仪 橙色延长线盘 应急灯",
|
||||
"expected_keywords": [
|
||||
"停电夜",
|
||||
"投影仪",
|
||||
"橙色延长线盘",
|
||||
"应急灯"
|
||||
],
|
||||
"minimum_keyword_hits": 2
|
||||
},
|
||||
{
|
||||
"query": "风铃塔观测 温湿度计 无糖姜茶",
|
||||
"expected_keywords": [
|
||||
"风铃塔观测",
|
||||
"温湿度计",
|
||||
"无糖姜茶",
|
||||
"校准"
|
||||
],
|
||||
"minimum_keyword_hits": 2
|
||||
},
|
||||
{
|
||||
"query": "东侧窗边 狸花猫 绿色硬壳笔记本 黄铜回形针",
|
||||
"expected_keywords": [
|
||||
"东侧窗边",
|
||||
"狸花猫",
|
||||
"绿色硬壳笔记本",
|
||||
"黄铜回形针"
|
||||
],
|
||||
"minimum_keyword_hits": 2
|
||||
}
|
||||
],
|
||||
"time_cases": [
|
||||
{
|
||||
"query": "投影仪 应急灯",
|
||||
"time_expression": "2026/03/06",
|
||||
"expected_keywords": [
|
||||
"投影仪",
|
||||
"应急灯",
|
||||
"橙色延长线盘"
|
||||
]
|
||||
},
|
||||
{
|
||||
"query": "温湿度计 姜茶",
|
||||
"time_expression": "2026/03/10",
|
||||
"expected_keywords": [
|
||||
"温湿度计",
|
||||
"无糖姜茶",
|
||||
"风铃塔观测"
|
||||
]
|
||||
},
|
||||
{
|
||||
"query": "狸花猫 绿色笔记本",
|
||||
"time_expression": "2026/03/14",
|
||||
"expected_keywords": [
|
||||
"狸花猫",
|
||||
"绿色硬壳笔记本",
|
||||
"黄铜回形针"
|
||||
]
|
||||
}
|
||||
],
|
||||
"episode_cases": [
|
||||
{
|
||||
"query": "那次值班柜第二层重新补物资的经过",
|
||||
"expected_keywords": [
|
||||
"值班柜第二层",
|
||||
"备用钥匙",
|
||||
"黑柄折叠伞",
|
||||
"创可贴"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"query": "停电夜抢救投影仪的经过",
|
||||
"expected_keywords": [
|
||||
"停电夜",
|
||||
"投影仪",
|
||||
"应急灯",
|
||||
"橙色延长线盘"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"query": "风铃塔观测前准备姜茶和温湿度计的那次安排",
|
||||
"expected_keywords": [
|
||||
"风铃塔观测",
|
||||
"温湿度计",
|
||||
"无糖姜茶",
|
||||
"硬板夹"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"query": "东侧窗边那只狸花猫闯进档案室那次",
|
||||
"expected_keywords": [
|
||||
"东侧窗边",
|
||||
"狸花猫",
|
||||
"绿色硬壳笔记本",
|
||||
"黄铜回形针"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
}
|
||||
],
|
||||
"knowledge_fetcher_cases": [
|
||||
{
|
||||
"query": "群里后来把值班柜第二层固定放哪些东西？",
|
||||
"expected_keywords": [
|
||||
"值班柜第二层",
|
||||
"备用钥匙",
|
||||
"黑柄折叠伞",
|
||||
"创可贴"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"query": "停电那次救投影仪时先找的是什么，延长线后来怎么规定？",
|
||||
"expected_keywords": [
|
||||
"应急灯",
|
||||
"仪器桌下层",
|
||||
"橙色延长线盘",
|
||||
"投影相关设备"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"query": "谁会在观测前校准温湿度计，还只喝无糖姜茶？",
|
||||
"expected_keywords": [
|
||||
"唐未",
|
||||
"温湿度计",
|
||||
"校准",
|
||||
"无糖姜茶"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
}
|
||||
],
|
||||
"profile_cases": [
|
||||
{
|
||||
"person_id": "6c4a50b4f4f34bdbb1a7a1c1e56c9001",
|
||||
"expected_keywords": [
|
||||
"林澈",
|
||||
"空调直吹",
|
||||
"二档",
|
||||
"桂花乌龙"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"person_id": "cbfd79d4680849b5ac7e23a3f6f09002",
|
||||
"expected_keywords": [
|
||||
"唐未",
|
||||
"温湿度计",
|
||||
"校准",
|
||||
"无糖姜茶"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"person_id": "a6eb73d41251472a8d229f1202df9003",
|
||||
"expected_keywords": [
|
||||
"周枝",
|
||||
"值班柜第二层",
|
||||
"备用钥匙",
|
||||
"黑柄折叠伞"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
}
|
||||
],
|
||||
"negative_control_cases": [
|
||||
{
|
||||
"query": "海盐柠檬饼干 原味苏打 辣味海苔 无糖薄荷糖",
|
||||
"source_batch_id": "snack_gossip_march_15_negative",
|
||||
"expected_behavior": "should_return_no_hits_if_only_positive_batches_are_ingested",
|
||||
"reason": "当前设计要求没有 Bot 发言的群聊批次不应进入长期记忆总结主路径。"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,862 @@
|
||||
{
|
||||
"meta": {
|
||||
"scenario_id": "group_chat_stream_memory_benchmark_hard",
|
||||
"description": "第二套更长、更刁钻的群聊聊天流长期记忆量化评估数据，刻意加入跨话题重叠词、自然句检索、长时段触发流以及更容易被群聊摘要淹没的人物事实。",
|
||||
"designed_for": [
|
||||
"group_chat",
|
||||
"topic_batching",
|
||||
"bot_participation_gate",
|
||||
"runtime_trigger",
|
||||
"search_mode",
|
||||
"time_mode",
|
||||
"episode_mode",
|
||||
"aggregate_mode",
|
||||
"person_fact_writeback",
|
||||
"negative_control",
|
||||
"cross_topic_overlap",
|
||||
"hard_mode"
|
||||
],
|
||||
"quantitative_targets": {
|
||||
"search": {
|
||||
"accuracy_at_1": 0.4,
|
||||
"recall_at_5": 0.75,
|
||||
"keyword_recall_at_5": 0.85
|
||||
},
|
||||
"knowledge_fetcher": {
|
||||
"success_rate": 0.75,
|
||||
"keyword_recall": 0.8
|
||||
},
|
||||
"profile": {
|
||||
"success_rate": 0.67,
|
||||
"evidence_rate": 1.0
|
||||
},
|
||||
"writeback": {
|
||||
"success_rate": 0.67,
|
||||
"keyword_recall": 0.8
|
||||
},
|
||||
"episode": {
|
||||
"success_rate": 0.75,
|
||||
"keyword_recall": 0.8
|
||||
},
|
||||
"negative_control": {
|
||||
"zero_hit_rate": 1.0
|
||||
},
|
||||
"runtime_trigger": {
|
||||
"positive_trigger_rate": 1.0,
|
||||
"negative_discard_rate": 1.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"session": {
|
||||
"session_id": "qq_group_535353",
|
||||
"platform": "qq",
|
||||
"user_id": "10010",
|
||||
"group_id": "535353",
|
||||
"display_name": "岚桥夜航协作群"
|
||||
},
|
||||
"import_payload": {
|
||||
"paragraphs": [
|
||||
{
|
||||
"content": "北塔夹层药箱抽屉通常固定放铜牌备用钥匙、银色保温毯、薄荷膏和黑色丁腈手套，夜间临时取用后需要在群里补报用途与归还时间。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark_hard",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"北塔夹层药箱",
|
||||
"铜牌备用钥匙",
|
||||
"银色保温毯",
|
||||
"薄荷膏",
|
||||
"黑色丁腈手套"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "雨棚工具架附近的短波电台返潮时，大家通常会先把机身擦干，再把黑色盘线和备用听筒重新卷回灰色防潮箱，最后更换新的硅胶包。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark_hard",
|
||||
"knowledge_type": "narrative",
|
||||
"entities": [
|
||||
"短波电台",
|
||||
"黑色盘线",
|
||||
"灰色防潮箱",
|
||||
"硅胶包",
|
||||
"备用听筒"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "晨雾采样结束后，蓝盖保温箱里的冷敷袋、编号试剂架和玻璃记号笔需要按固定顺序归位，避免交接时把样本编号和温控记录混在一起。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark_hard",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"蓝盖保温箱",
|
||||
"冷敷袋",
|
||||
"编号试剂架",
|
||||
"玻璃记号笔",
|
||||
"样本编号"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "西廊风口观测前，迟雨会先校准气压计并核对湿度夹板，桌边常备温梨汤和风向丝带，避免记录员在长风口里来回找物件。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark_hard",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"迟雨",
|
||||
"气压计",
|
||||
"湿度夹板",
|
||||
"温梨汤",
|
||||
"风向丝带"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "沈砚值夜时不喜欢冷白荧光灯直照，常把小风扇固定在三档，并且会自带盐渍梅子气泡水放在个人层架上。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark_hard",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"沈砚",
|
||||
"冷白荧光灯",
|
||||
"小风扇三档",
|
||||
"盐渍梅子气泡水"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "贺岚平时看管北塔夹层药箱和铜牌备用钥匙，取用后会顺手补回黑色丁腈手套，并检查银色保温毯有没有被塞反。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark_hard",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"贺岚",
|
||||
"北塔夹层药箱",
|
||||
"铜牌备用钥匙",
|
||||
"黑色丁腈手套",
|
||||
"银色保温毯"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "绘图室天窗边长桌常压着象牙描图纸、琥珀夹子和黄铜镇纸，白耳鸮闯入时最容易把这一角的纸页和夹具掀乱。",
|
||||
"source": "fixture:group_chat_stream_memory_benchmark_hard",
|
||||
"knowledge_type": "narrative",
|
||||
"entities": [
|
||||
"绘图室天窗边",
|
||||
"象牙描图纸",
|
||||
"琥珀夹子",
|
||||
"黄铜镇纸",
|
||||
"白耳鸮"
|
||||
]
|
||||
}
|
||||
],
|
||||
"relations": [
|
||||
{
|
||||
"subject": "贺岚",
|
||||
"predicate": "看管",
|
||||
"object": "北塔夹层药箱"
|
||||
},
|
||||
{
|
||||
"subject": "北塔夹层药箱",
|
||||
"predicate": "存放",
|
||||
"object": "铜牌备用钥匙"
|
||||
},
|
||||
{
|
||||
"subject": "北塔夹层药箱",
|
||||
"predicate": "存放",
|
||||
"object": "银色保温毯"
|
||||
},
|
||||
{
|
||||
"subject": "顾澄",
|
||||
"predicate": "维护",
|
||||
"object": "短波电台"
|
||||
},
|
||||
{
|
||||
"subject": "迟雨",
|
||||
"predicate": "负责",
|
||||
"object": "气压计校准"
|
||||
},
|
||||
{
|
||||
"subject": "迟雨",
|
||||
"predicate": "偏好",
|
||||
"object": "温梨汤"
|
||||
},
|
||||
{
|
||||
"subject": "沈砚",
|
||||
"predicate": "不喜欢",
|
||||
"object": "冷白荧光灯直照"
|
||||
},
|
||||
{
|
||||
"subject": "绘图室天窗边",
|
||||
"predicate": "摆放",
|
||||
"object": "象牙描图纸"
|
||||
}
|
||||
]
|
||||
},
|
||||
"simulated_stream_batches": [
|
||||
{
|
||||
"batch_id": "mezzanine_first_aid_march_21",
|
||||
"topic": "北塔夹层药箱补位",
|
||||
"bot_participated": true,
|
||||
"expected_behavior": "should_be_summarized",
|
||||
"message_count": 12,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"沈砚",
|
||||
"贺岚",
|
||||
"迟雨",
|
||||
"顾澄"
|
||||
],
|
||||
"start_time": 1774095120.0,
|
||||
"end_time": 1774098180.0,
|
||||
"messages": [
|
||||
"[2026-03-21 20:12] 沈砚：北塔值夜的冷白灯又直打眼睛，我把小风扇拨回三档时顺手翻了一下夹层药箱，铜牌备用钥匙还在，但银色保温毯和薄荷膏的位置都乱了。",
|
||||
"[2026-03-21 20:14] 贺岚：黑色丁腈手套昨晚急救演练拿走了两副，我还没来得及补，药箱里左边那格现在确实空得很明显。",
|
||||
"[2026-03-21 20:18] 顾澄：旧清单上写的是铜牌备用钥匙、银色保温毯、薄荷膏、黑色丁腈手套和哨卡，但哨卡早改到门侧挂袋了。",
|
||||
"[2026-03-21 20:22] Mai：那今晚重新定一遍，夹层药箱只保留铜牌备用钥匙、银色保温毯、薄荷膏和黑色丁腈手套，谁临时拿走都在群里补报用途和预计归还时间。",
|
||||
"[2026-03-21 20:27] 迟雨：薄荷膏只剩底，保温毯边角也卷了，最好一并补新，不然真遇到低温处理时手忙脚乱。",
|
||||
"[2026-03-21 20:31] 沈砚：我半夜最怕一边被灯晃一边找物资，所以钥匙和保温毯的位置最好别再动来动去，我自己的盐渍梅子气泡水也不会放抽屉里。",
|
||||
"[2026-03-21 20:36] 贺岚：我待会儿带一盒新薄荷膏和两包丁腈手套过去，顺便把保温毯重新折成外翻口，黑里摸会更快。",
|
||||
"[2026-03-21 20:40] 顾澄：铜牌备用钥匙建议继续夹中间，别再和创伤贴挤在一起，上次取钥匙把整包小药片都带出来了。",
|
||||
"[2026-03-21 20:45] Mai：我记成新版摆放顺序了，左保温毯，中央铜牌备用钥匙，右边薄荷膏和丁腈手套；个人饮料和私人物件一律不进抽屉。",
|
||||
"[2026-03-21 20:49] 迟雨：如果临时取保温毯或手套，至少发“已取用+用途+归还时间”，不要只在走廊口头说一声。",
|
||||
"[2026-03-21 20:56] 贺岚：补完我会拍张抽屉照片，连折法和钥匙朝向一起拍进去，之后新值夜的人直接看图就行。",
|
||||
"[2026-03-21 21:03] Mai：就按这版执行，后面谁要改夹层药箱布局，先在群里说，不然下次夜班还是会翻半天。"
|
||||
],
|
||||
"combined_text": "北塔夹层药箱这一轮群聊把铜牌备用钥匙、银色保温毯、薄荷膏、黑色丁腈手套的摆放与补位规则重新说得很细。大家先从沈砚值夜时被冷白荧光灯直照、顺手检查抽屉说起，接着确认旧清单、演练后没有补回的丁腈手套、卷边的银色保温毯和几乎见底的薄荷膏到底该放在哪一格。Mai 明确要求夹层药箱只保留四样公用物资，谁取用都要在群里补报用途与归还时间；贺岚负责补回手套和薄荷膏，并承诺拍清楚抽屉照片；顾澄补充钥匙不能再与小药片混放；沈砚再次提到自己值夜时会把小风扇固定在三档，个人带来的盐渍梅子气泡水不会放进公用抽屉里。后半段群聊又进一步确认了左保温毯、中间钥匙、右侧药品的最终布局，说明旧清单中的哨卡已经迁出，不应再占用抽屉空间，并强调夜里应急时谁先摸到物资、谁负责在群里补一句用途和归还时间。整段对话同时覆盖了物资补位、摆放顺序、夜间可见性、旧规则迁移和报备流程，是一段标准而冗长的群聊收束样本。",
|
||||
"expected_memory_targets": [
|
||||
"北塔夹层药箱",
|
||||
"铜牌备用钥匙",
|
||||
"银色保温毯",
|
||||
"薄荷膏",
|
||||
"黑色丁腈手套",
|
||||
"报备"
|
||||
]
|
||||
},
|
||||
{
|
||||
"batch_id": "rain_shed_radio_march_23",
|
||||
"topic": "雨棚短波电台返潮排查",
|
||||
"bot_participated": true,
|
||||
"expected_behavior": "should_be_summarized",
|
||||
"message_count": 12,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"顾澄",
|
||||
"贺岚",
|
||||
"迟雨",
|
||||
"沈砚"
|
||||
],
|
||||
"start_time": 1774275240.0,
|
||||
"end_time": 1774278360.0,
|
||||
"messages": [
|
||||
"[2026-03-23 22:14] 顾澄：雨棚那台短波电台今晚一开机就有潮噪，灰色防潮箱外壳摸着也冷，黑色盘线有一截像是沾了雾水。",
|
||||
"[2026-03-23 22:16] 贺岚：备用听筒没事，主要是箱里那包硅胶已经发软了，估计这两天夜里返潮比想的更重。",
|
||||
"[2026-03-23 22:20] 沈砚：别直接通电硬顶，我先把台面上的小黄灯关掉再看接口，反光太重会误判，盘线最好先拆下来擦干。",
|
||||
"[2026-03-23 22:26] Mai：按顺序处理，顾澄先把机身外壳擦干，贺岚把黑色盘线和备用听筒摊开，迟雨看一下灰色防潮箱底部的吸水毡有没有饱和。",
|
||||
"[2026-03-23 22:31] 迟雨：吸水毡已经发黏，箱底还有一点冷凝水，硅胶包也该整包换新，不然把盘线卷回去还是会闷出潮气。",
|
||||
"[2026-03-23 22:35] 顾澄：机身擦干后杂音小了一半，问题大概不是主板，是存放时盘线缠得太紧，潮气都闷在接头这圈。",
|
||||
"[2026-03-23 22:40] 贺岚：我把黑色盘线全部放开了，备用听筒也擦过一遍，灰色防潮箱里原来的旧硅胶包先扔掉。",
|
||||
"[2026-03-23 22:45] Mai：之后统一改成机身先晾五分钟、盘线松卷、听筒单独装袋、灰色防潮箱最后再关盖，别再图快一股脑塞进去。",
|
||||
"[2026-03-23 22:50] 沈砚：雨棚边夜里冷风大，机身外壳要是还冒凉气就别急着合盖，明早又得重新返潮。",
|
||||
"[2026-03-23 22:56] 迟雨：新硅胶包已经放进去两袋，吸水毡我改成倒放在最底层，至少不会直接顶着电台外壳。",
|
||||
"[2026-03-23 23:01] 顾澄：我把黑色盘线重新分成两圈，接头朝上，短波电台和备用听筒都回箱了，现在基本没有潮噪。",
|
||||
"[2026-03-23 23:06] Mai：结论记一下，雨棚短波电台返潮时先擦机身、松黑色盘线、换硅胶包，再按灰色防潮箱的新顺序归位。"
|
||||
],
|
||||
"combined_text": "雨棚短波电台这段对话比一般设备检修更刁钻，因为大家反复讨论的是返潮噪声、灰色防潮箱底部冷凝水、黑色盘线缠绕方式和硅胶包失效这些容易被混成同类词的细节。顾澄先发现短波电台有潮噪，贺岚确认备用听筒没坏而是防潮箱和硅胶包出了问题，沈砚提醒不要在反光和冷风里急着通电，Mai 则把处理顺序拆成机身擦干、盘线摊开、吸水毡检查和硅胶包更换。迟雨进一步指出灰色防潮箱底部的吸水毡已经发黏，需要调整放置方式。后面的讨论还涉及盘线要不要分成两圈、接头是否朝上、机身晾多久再合盖、备用听筒需不需要单独装袋，以及第二天谁来复查灰色防潮箱里的湿气状态。最后大家把短波电台、黑色盘线、备用听筒和灰色防潮箱的归位规则重新说清楚，让这一段既像设备维护，又像存放流程重整，天然适合测试跨话题重叠词下的检索稳定性。",
|
||||
"expected_memory_targets": [
|
||||
"短波电台",
|
||||
"灰色防潮箱",
|
||||
"黑色盘线",
|
||||
"硅胶包",
|
||||
"备用听筒"
|
||||
]
|
||||
},
|
||||
{
|
||||
"batch_id": "fog_transfer_march_26",
|
||||
"topic": "晨雾采样蓝盖保温箱排序",
|
||||
"bot_participated": true,
|
||||
"expected_behavior": "should_be_summarized",
|
||||
"message_count": 12,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"迟雨",
|
||||
"顾澄",
|
||||
"贺岚",
|
||||
"沈砚"
|
||||
],
|
||||
"start_time": 1774475280.0,
|
||||
"end_time": 1774478520.0,
|
||||
"messages": [
|
||||
"[2026-03-26 05:48] 迟雨：晨雾采样刚收尾，蓝盖保温箱里冷敷袋、编号试剂架和玻璃记号笔被塞反了，后面交接的人可能一打开就拿错。",
|
||||
"[2026-03-26 05:52] 顾澄：我刚把样本管临时放回去时太赶了，冷敷袋压在编号试剂架上面了，玻璃记号笔也滑到最底层去了。",
|
||||
"[2026-03-26 05:57] 贺岚：吸水垫已经湿透一半，保温箱里要是再混着放，样本编号和温控记录很容易一起糊掉。",
|
||||
"[2026-03-26 06:02] Mai：先别再往里塞东西，按新版顺序来，最下层冷敷袋，中层编号试剂架，右侧槽放玻璃记号笔和封口贴，样本管最后再扣。",
|
||||
"[2026-03-26 06:08] 沈砚：我先把外侧工作灯压低一点，冷白灯映在箱盖上太亮，我看编号时总会反光，看错一位就麻烦。",
|
||||
"[2026-03-26 06:13] 迟雨：我把蓝盖保温箱里的冷敷袋重新平码了，编号试剂架按奇偶分左右，这样交接时一眼就能扫清楚。",
|
||||
"[2026-03-26 06:18] 顾澄：玻璃记号笔现在固定在右槽，不再横放，我顺手把备用封口贴也放在笔旁边了。",
|
||||
"[2026-03-26 06:24] 贺岚：样本管外壁的雾水有点重，我加了一层薄吸水纸，不然冷敷袋融水会一路蹭到标签。",
|
||||
"[2026-03-26 06:30] Mai：之后凌晨交接都照这个次序，谁要临时改动蓝盖保温箱内部顺序，也得在群里补一句，免得下一班以为还是旧布局。",
|
||||
"[2026-03-26 06:35] 迟雨：我已经把编号试剂架和样本位次对应关系写到箱盖内侧了，等会儿拍照发群文件。",
|
||||
"[2026-03-26 06:39] 沈砚：这样就算我半困着看，也不至于拿错玻璃记号笔或者把冷敷袋从最上层一把抽出来。",
|
||||
"[2026-03-26 06:42] Mai：确认一下，本次晨雾采样交接按蓝盖保温箱新顺序执行，冷敷袋、编号试剂架、玻璃记号笔和样本管都不要再混层。"
|
||||
],
|
||||
"combined_text": "晨雾采样这一段专门压在“凌晨交接、低温、反光、编号顺序”这些容易让检索混淆的词上。大家围绕蓝盖保温箱里冷敷袋、编号试剂架、玻璃记号笔和样本管的归位顺序聊得很细，从顾澄一开始匆忙把冷敷袋压在试剂架上，到贺岚担心吸水垫、标签和融水互相污染，再到 Mai 明确规定最下层冷敷袋、中层编号试剂架、右槽玻璃记号笔和封口贴、样本管最后扣紧。沈砚额外提到冷白灯反光会影响编号辨认，迟雨把架位关系写到箱盖内侧并承诺拍照存档。随后大家又补充了吸水纸、备用封口贴、样本管雾水、奇偶编号左右分层和凌晨换班时必须按箱盖顺序复核这些细节，让这一段不仅仅是物品清单，而是包含低温保存、交接动作、标签保护和视觉干扰的完整操作窗口。整段话题兼有物品清单、流程顺序、交接规则和环境噪声，非常适合验证更长文本下的 search、time 和 episode 区分能力。",
|
||||
"expected_memory_targets": [
|
||||
"蓝盖保温箱",
|
||||
"冷敷袋",
|
||||
"编号试剂架",
|
||||
"玻璃记号笔",
|
||||
"样本管"
|
||||
]
|
||||
},
|
||||
{
|
||||
"batch_id": "west_corridor_wind_march_29",
|
||||
"topic": "西廊风口观测准备",
|
||||
"bot_participated": true,
|
||||
"expected_behavior": "should_be_summarized",
|
||||
"message_count": 12,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"迟雨",
|
||||
"沈砚",
|
||||
"许窈",
|
||||
"贺岚"
|
||||
],
|
||||
"start_time": 1774779720.0,
|
||||
"end_time": 1774783080.0,
|
||||
"messages": [
|
||||
"[2026-03-29 18:22] 迟雨：今晚西廊风口比预想更急，我先把气压计拿出来校准，湿度夹板也得重新夹一张干纸，不然一小时后全卷边。",
|
||||
"[2026-03-29 18:25] 沈砚：工作台这边冷白灯太刺，我先换成侧边黄灯，再把小风扇关小一点，不然气流会把风向丝带吹得乱抖。",
|
||||
"[2026-03-29 18:29] 许窈：温梨汤我装到窄口保温壶里了，放在西廊门内侧靠墙的位置，别和公用记录板挤在一起。",
|
||||
"[2026-03-29 18:34] 贺岚：风向丝带刚整理过一遍，我把备用夹子和细线都放到记录板后面的小袋里，等会儿就不用来回摸。",
|
||||
"[2026-03-29 18:39] Mai：流程再说一遍，迟雨先报气压计校准完成，许窈确认温梨汤和备用夹子位置，沈砚负责灯光和风扇不要干扰风口读数。",
|
||||
"[2026-03-29 18:44] 迟雨：气压计现在回到零点附近了，湿度夹板也夹好了新纸，等第一轮风向记录时我再复核一次。",
|
||||
"[2026-03-29 18:48] 沈砚：黄灯角度调好了，这样不会直照读数盘面，我自己的盐渍梅子气泡水也还是放个人架，不会碰公用桌面。",
|
||||
"[2026-03-29 18:53] 许窈：温梨汤和备用杯已经摆好，门口风太直，我给壶套了防滑圈，谁喝完别顺手放回记录夹旁边。",
|
||||
"[2026-03-29 18:58] 贺岚：风向丝带试摆过了，夹点不动，细线也没打结，记录板后袋里还有两枚备用夹子。",
|
||||
"[2026-03-29 19:04] Mai：今晚西廊风口观测前的关键点就三件，气压计校准、湿度夹板换纸、温梨汤和夹具别混放，之后都照这版执行。",
|
||||
"[2026-03-29 19:11] 迟雨：收到，我把“先校准气压计再开记录”写到第一行了，后面换班的人也不用再问一次。",
|
||||
"[2026-03-29 19:18] Mai：这段我会记进观测前准备说明里，尤其是气压计、湿度夹板、风向丝带和温梨汤的位置别再临时换。"
|
||||
],
|
||||
"combined_text": "西廊风口观测准备这段特意把人物偏好和观测流程混在一起，模拟真实群聊里“事实很容易被话题背景淹没”的情况。迟雨一直在谈气压计校准和湿度夹板换纸，许窈负责温梨汤和备用杯摆位，贺岚补充风向丝带、备用夹子和细线的位置，沈砚则反复抱怨冷白灯太刺眼、改用侧边黄灯并控制小风扇不要干扰读数。Mai 把整套流程收束成观测前三步，并强调温梨汤、夹具和记录板不能混放。后续大家又把保温壶的防滑圈、备用夹子分配、风向丝带试摆、个人饮料不能挤进公用桌面、以及第一轮记录前必须先报校准完成这些动作重复确认了一遍。这里既有人物稳定偏好，又有西廊风口这一特定场景和大量易重叠的工具名，还包含多轮确认与换班提示，是验证 person_fact 是否能真正压过群聊背景噪声的好素材。",
|
||||
"expected_memory_targets": [
|
||||
"西廊风口",
|
||||
"气压计",
|
||||
"湿度夹板",
|
||||
"温梨汤",
|
||||
"风向丝带"
|
||||
]
|
||||
},
|
||||
{
|
||||
"batch_id": "skylight_owl_april_02",
|
||||
"topic": "绘图室天窗白耳鸮闯入",
|
||||
"bot_participated": true,
|
||||
"expected_behavior": "should_be_summarized",
|
||||
"message_count": 12,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"许窈",
|
||||
"顾澄",
|
||||
"贺岚",
|
||||
"沈砚"
|
||||
],
|
||||
"start_time": 1775133060.0,
|
||||
"end_time": 1775136540.0,
|
||||
"messages": [
|
||||
"[2026-04-02 20:31] 许窈：绘图室天窗缝刚被风顶开，一只白耳鸮直接扑到长桌上，把象牙描图纸掀起来一大片，琥珀夹子全滑到地上了。",
|
||||
"[2026-04-02 20:34] 顾澄：别先追鸟，我先把黄铜镇纸按回纸角，不然描图纸卷得更厉害，边上的玻璃尺也快掉了。",
|
||||
"[2026-04-02 20:38] 贺岚：折梯在门后，我去把天窗先拉上，琥珀夹子我捡回一半了，有两枚滑到东侧柜脚下。",
|
||||
"[2026-04-02 20:42] 沈砚：先别开顶灯，那盏冷白灯一亮鸟会更乱撞，我把侧边小灯压低，别让它再扑向纸架。",
|
||||
"[2026-04-02 20:47] Mai：处理顺序先固定，顾澄压纸，贺岚关天窗找夹子，许窈把卷起的象牙描图纸按编号重新叠回去，不要一边追鸟一边挪桌上工具。",
|
||||
"[2026-04-02 20:51] 许窈：最上面的两张描图纸已经有折痕了，但编号还在，我先用黄铜镇纸压住，再找少掉的琥珀夹子。",
|
||||
"[2026-04-02 20:56] 顾澄：白耳鸮现在站到东梁上了，暂时不碰桌子，玻璃尺和铅笔盒我已经移到靠墙这边，省得再被翅膀扫下去。",
|
||||
"[2026-04-02 21:00] 贺岚：天窗锁好了，东侧柜脚下捡回三枚琥珀夹子，另外一枚卡在桌脚和纸箱中间，我伸手就能拿到。",
|
||||
"[2026-04-02 21:05] 沈砚：侧灯角度够了，至少不会再直照到鸟和纸面，我这边风扇也先停了，免得象牙描图纸继续抖。",
|
||||
"[2026-04-02 21:10] Mai：等鸟自己往高处退，我们先把桌面恢复，象牙描图纸按编号叠好，琥珀夹子回左盒，黄铜镇纸继续压角，别因为一阵慌把原来的顺序全打散。",
|
||||
"[2026-04-02 21:18] 许窈：桌面基本恢复了，少掉的那枚夹子也找回来了，描图纸折痕部分我另放最上层，明天再单独压平。",
|
||||
"[2026-04-02 21:29] Mai：今晚的结论我记一下，绘图室天窗边长桌继续固定放象牙描图纸、琥珀夹子和黄铜镇纸，白耳鸮闯入时先稳纸再关窗。"
|
||||
],
|
||||
"combined_text": "白耳鸮闯进绘图室这一段专门拿来压 episode query 的自然语言理解，因为话题里既有事件性描述，又有大量静物摆放细节。大家先处理的是天窗缝被顶开后，象牙描图纸、琥珀夹子和黄铜镇纸被掀乱的问题；随后又讨论冷白灯会刺激白耳鸮、侧灯角度、风扇是否需要停、折梯和玻璃尺的位置、以及描图纸折痕如何单独压平。Mai 没有只说‘把鸟赶出去’，而是明确总结成‘先稳纸再关窗’，并恢复了绘图室天窗边长桌的固定摆放规则。后半段群聊还补了夹子掉落点、东侧柜脚、桌脚纸箱缝、顶灯和侧灯的切换、以及第二天如何单独压平最上层折痕纸这些容易被普通检索忽略的细枝末节。这类又长又绕、既有事件又有物件的群聊，非常适合验证我们刚修过的中文自然句 episode 检索是否真的泛化。",
|
||||
"expected_memory_targets": [
|
||||
"白耳鸮",
|
||||
"象牙描图纸",
|
||||
"琥珀夹子",
|
||||
"黄铜镇纸",
|
||||
"天窗"
|
||||
]
|
||||
},
|
||||
{
|
||||
"batch_id": "dessert_idle_april_04_negative",
|
||||
"topic": "夜宵甜点口味闲聊",
|
||||
"bot_participated": false,
|
||||
"expected_behavior": "ignored_by_summarizer_without_bot_message",
|
||||
"message_count": 10,
|
||||
"participants": [
|
||||
"许窈",
|
||||
"顾澄",
|
||||
"迟雨",
|
||||
"沈砚"
|
||||
],
|
||||
"start_time": 1775301000.0,
|
||||
"end_time": 1775304120.0,
|
||||
"messages": [
|
||||
"[2026-04-04 19:10] 许窈：我刚在街口买到荔枝冻，里面还有一点桂花糖浆，冰得很夸张。",
|
||||
"[2026-04-04 19:14] 顾澄：我更想吃焦糖海盐脆片，那种一咬就掉屑的比糯口点心适合夜里配热水。",
|
||||
"[2026-04-04 19:19] 迟雨：乌龙奶酥我也喜欢，但甜得太快，还是百香果软糖更耐嚼。",
|
||||
"[2026-04-04 19:24] 沈砚：夜里我其实只想吃不黏手的，海盐脆片和柚子糖都比奶酥方便。",
|
||||
"[2026-04-04 19:31] 许窈：那我下次把荔枝冻换成常温的，不然冰箱味太重。",
|
||||
"[2026-04-04 19:37] 顾澄：要是有烤糯米团就好了，焦糖和海盐一起裹会很香。",
|
||||
"[2026-04-04 19:44] 迟雨：百香果软糖一定要选酸一点的，不然吃两颗就发腻。",
|
||||
"[2026-04-04 19:51] 沈砚：乌龙奶酥掉渣太厉害，我更想找一盒柠檬脆片。",
|
||||
"[2026-04-04 19:58] 许窈：行，那我下次夜宵就别买太黏的，省得桌上全是糖粉。",
|
||||
"[2026-04-04 20:02] 顾澄：决定了，下一轮就试荔枝冻、海盐脆片和百香果软糖三选二。"
|
||||
],
|
||||
"combined_text": "这一批是刻意保留下来的负样本，整段只围绕荔枝冻、焦糖海盐脆片、乌龙奶酥、百香果软糖和柚子糖的口味闲聊，没有 Mai 参与，也没有任何需要被长期记忆化的群聊任务结论。它的设计目的不是构造一个完全短小的无聊片段，而是制造一段在词面上同样丰富、长度也足够的闲聊，以便验证 summarizer 的 Bot 参与门槛和检索层的误召回控制是否真的有效。",
|
||||
"expected_memory_targets": [
|
||||
"荔枝冻",
|
||||
"焦糖海盐脆片",
|
||||
"乌龙奶酥",
|
||||
"百香果软糖"
|
||||
]
|
||||
}
|
||||
],
|
||||
"runtime_trigger_streams": [
|
||||
{
|
||||
"stream_id": "runtime_west_corridor_trigger_april_07",
|
||||
"topic": "西廊风口准备长流触发样本",
|
||||
"trigger_mode": "time_threshold",
|
||||
"elapsed_since_last_check_hours": 8.8,
|
||||
"bot_participated": true,
|
||||
"expected_check_outcome": "should_trigger_topic_check_and_pass_bot_gate",
|
||||
"expected_next_stage": "topic_cache_should_update",
|
||||
"message_count": 22,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"迟雨",
|
||||
"沈砚",
|
||||
"许窈",
|
||||
"贺岚"
|
||||
],
|
||||
"start_time": 1775523960.0,
|
||||
"end_time": 1775555880.0,
|
||||
"messages": [
|
||||
"[2026-04-07 09:06] 迟雨：今早西廊风口比昨晚更急，气压计零点又飘了一点，我先不急着开记录。",
|
||||
"[2026-04-07 09:18] 沈砚：门边冷白灯太刺，我把黄灯转过去一点，不然表盘反光还是读不清。",
|
||||
"[2026-04-07 09:27] 许窈：温梨汤已经装进新壶了，先放在门内侧，不和记录板摆一排。",
|
||||
"[2026-04-07 09:41] 贺岚：备用夹子和风向丝带我都放进后袋了，细线也重新绕好了。",
|
||||
"[2026-04-07 10:03] Mai：今天沿用西廊风口旧流程，先校准气压计，再换湿度夹板的纸，最后确认温梨汤和夹具位置。",
|
||||
"[2026-04-07 10:17] 迟雨：气压计现在回稳了，但湿度夹板上的旧纸边角已经卷起来，最好直接重夹。",
|
||||
"[2026-04-07 10:33] 沈砚：我把风扇关小了，别让风向丝带被室内气流带偏。",
|
||||
"[2026-04-07 10:57] 许窈：保温壶外壁有点滑，我又套了防滑圈，省得一碰就滚到门口去。",
|
||||
"[2026-04-07 11:15] 贺岚：记录板后袋里的夹子还够，两枚大的留给风向丝带，细夹子给湿度纸。",
|
||||
"[2026-04-07 11:39] Mai：谁临时挪了气压计或湿度夹板，记得在群里说一声，别让下一班找不到。",
|
||||
"[2026-04-07 12:08] 迟雨：第一轮记录结束，我把气压计位置和校准时间写到夹板右上角了。",
|
||||
"[2026-04-07 12:44] 沈砚：黄灯角度再压低一点就够了，现在表盘基本没有白斑。",
|
||||
"[2026-04-07 13:11] 许窈：温梨汤还热，我又加了一个空杯，下午那班不用再回去找。",
|
||||
"[2026-04-07 13:43] 贺岚：风向丝带其中一根打了结，我已经拆开重绑，备用细线也补了一卷。",
|
||||
"[2026-04-07 14:06] Mai：下午继续按这版，不要把温梨汤放到公用记录夹旁边，也别让个人饮料挤进工作台。",
|
||||
"[2026-04-07 14:38] 迟雨：第二轮之前我再复核一次气压计，湿度夹板的纸目前还是干的。",
|
||||
"[2026-04-07 15:09] 沈砚：我自己的盐渍梅子气泡水还是放个人架，公用桌上只留气压计和夹板。",
|
||||
"[2026-04-07 15:37] 许窈：门口风更大了，保温壶我往里挪了半步，但还是在原来的内侧角落。",
|
||||
"[2026-04-07 16:03] 贺岚：备用夹子数量没变，风向丝带和细线现在都在后袋左边。",
|
||||
"[2026-04-07 16:41] Mai：我把“先校准气压计再开记录”的提示写到最上面了，换班的人照着做就行。",
|
||||
"[2026-04-07 17:12] 迟雨：第三轮读数也稳定，湿度夹板今天的纸还能撑到收工。",
|
||||
"[2026-04-07 17:58] Mai：今天西廊风口准备流程就按这版固化，气压计、湿度夹板、风向丝带和温梨汤都别再临时换位。"
|
||||
],
|
||||
"expected_memory_targets": [
|
||||
"西廊风口",
|
||||
"气压计",
|
||||
"湿度夹板",
|
||||
"温梨汤",
|
||||
"风向丝带"
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream_id": "runtime_skylight_trigger_april_09",
|
||||
"topic": "白耳鸮闯入长流触发样本",
|
||||
"trigger_mode": "time_threshold",
|
||||
"elapsed_since_last_check_hours": 8.9,
|
||||
"bot_participated": true,
|
||||
"expected_check_outcome": "should_trigger_topic_check_and_pass_bot_gate",
|
||||
"expected_next_stage": "topic_cache_should_update",
|
||||
"message_count": 22,
|
||||
"participants": [
|
||||
"Mai",
|
||||
"许窈",
|
||||
"顾澄",
|
||||
"贺岚",
|
||||
"沈砚"
|
||||
],
|
||||
"start_time": 1775707860.0,
|
||||
"end_time": 1775739780.0,
|
||||
"messages": [
|
||||
"[2026-04-09 12:11] 许窈：中午检窗时发现绘图室天窗锁没扣紧，我先把象牙描图纸压回去，免得下午又起风。",
|
||||
"[2026-04-09 12:25] 顾澄：琥珀夹子今天早上少了一枚，我怀疑还卡在桌脚边。",
|
||||
"[2026-04-09 12:42] 贺岚：折梯已经挪回门后，等会儿再顺手看一眼天窗扣件。",
|
||||
"[2026-04-09 13:06] 沈砚：侧灯现在正常，顶灯还是先别开，冷白光一照纸面就晃得厉害。",
|
||||
"[2026-04-09 13:31] Mai：先把桌面顺序恢复，象牙描图纸、琥珀夹子和黄铜镇纸都照旧摆，天窗问题下午统一处理。",
|
||||
"[2026-04-09 14:02] 许窈：刚刚那只白耳鸮又落回外沿了，好在没再扑进来，我先不惊它。",
|
||||
"[2026-04-09 14:29] 顾澄：少的那枚夹子找到了，果然卡在桌脚和纸箱中间。",
|
||||
"[2026-04-09 14:58] 贺岚：天窗扣件我试了一下，有一边发涩，合上时得多推半格。",
|
||||
"[2026-04-09 15:21] 沈砚：风扇先停吧，纸边已经有点抖，别把描图纸又吹翘了。",
|
||||
"[2026-04-09 15:46] Mai：如果白耳鸮再闯进来，还是按先稳纸、再关窗、最后收桌面工具的顺序来。",
|
||||
"[2026-04-09 16:03] 许窈：我把折痕最明显的两张描图纸单独压在最上面，明天再慢慢压平。",
|
||||
"[2026-04-09 16:24] 顾澄：黄铜镇纸这边够用，四角都压住了，玻璃尺也移到了靠墙那边。",
|
||||
"[2026-04-09 16:52] 贺岚：门后的折梯没再动，天窗如果晚上还发涩，我明早再上去调。",
|
||||
"[2026-04-09 17:15] 沈砚：侧灯角度现在刚好，不会直打鸟也不会晃到纸面。",
|
||||
"[2026-04-09 17:41] Mai：晚班别改桌面布局，尤其别把琥珀夹子和黄铜镇纸分开放，不然一慌又找不到。",
|
||||
"[2026-04-09 18:08] 许窈：描图纸编号已经重新核对完，缺页没有新增。",
|
||||
"[2026-04-09 18:39] 顾澄：桌脚周围也清出来了，之后夹子掉落会更容易捡。",
|
||||
"[2026-04-09 19:02] 贺岚：天窗锁现在能扣上，只是右边还是偏紧，先留个提醒。",
|
||||
"[2026-04-09 19:34] 沈砚：今晚继续用侧灯，冷白顶灯别开，省得纸面和鸟都受刺激。",
|
||||
"[2026-04-09 20:07] Mai：我把白耳鸮闯入时的桌面恢复顺序写进说明里了，谁值班都照着做。",
|
||||
"[2026-04-09 20:36] 许窈：象牙描图纸、琥珀夹子、黄铜镇纸都在固定位置，没有再乱。",
|
||||
"[2026-04-09 21:03] Mai：这波可以收尾了，绘图室天窗边长桌的固定摆放和应急顺序都不再改。"
|
||||
],
|
||||
"expected_memory_targets": [
|
||||
"白耳鸮",
|
||||
"象牙描图纸",
|
||||
"琥珀夹子",
|
||||
"黄铜镇纸",
|
||||
"天窗"
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream_id": "runtime_dessert_negative_april_11",
|
||||
"topic": "甜点闲聊长流负样本",
|
||||
"trigger_mode": "time_threshold",
|
||||
"elapsed_since_last_check_hours": 8.6,
|
||||
"bot_participated": false,
|
||||
"expected_check_outcome": "should_trigger_topic_check_but_be_discarded_without_bot_message",
|
||||
"expected_next_stage": "topic_cache_should_remain_empty",
|
||||
"message_count": 21,
|
||||
"participants": [
|
||||
"许窈",
|
||||
"顾澄",
|
||||
"迟雨",
|
||||
"沈砚"
|
||||
],
|
||||
"start_time": 1775884080.0,
|
||||
"end_time": 1775915160.0,
|
||||
"messages": [
|
||||
"[2026-04-11 13:08] 许窈：中午那盒荔枝冻居然还剩两块，桂花糖浆味道比昨天重，盒底那层糖水也比我记得更黏一些。",
|
||||
"[2026-04-11 13:22] 顾澄：我还是更想吃焦糖海盐脆片，起码不会晃一下就碎成汤。",
|
||||
"[2026-04-11 13:37] 迟雨：乌龙奶酥闻起来很香，但吃两口就太腻，百香果软糖反而更合适。",
|
||||
"[2026-04-11 13:55] 沈砚：软糖太黏了，我宁愿吃柚子脆片或者原味苏打。",
|
||||
"[2026-04-11 14:11] 许窈：荔枝冻如果不冰透其实不错，就是盒子一开就会流糖水。",
|
||||
"[2026-04-11 14:28] 顾澄：焦糖海盐和烤糯米片如果放一起，应该会比奶酥耐吃，至少不会一拆包装就满桌掉屑。",
|
||||
"[2026-04-11 14:46] 迟雨：百香果软糖最好选酸一点的，不然甜度太直白。",
|
||||
"[2026-04-11 15:02] 沈砚：乌龙奶酥掉渣太夸张，晚上值班根本不适合拿着吃。",
|
||||
"[2026-04-11 15:19] 许窈：那下次我换成荔枝冻和海盐脆片，至少不用到处拍碎屑。",
|
||||
"[2026-04-11 15:37] 顾澄：再加一包柠檬薄片吧，甜口之间吃一点会清爽很多。",
|
||||
"[2026-04-11 15:56] 迟雨：如果有酸梅软糖我也可以，和百香果轮着吃不会太腻。",
|
||||
"[2026-04-11 16:14] 沈砚：苏打饼和软糖混着吃也行，主要是别太黏手，不然摸完包装还得再找纸擦。",
|
||||
"[2026-04-11 16:33] 许窈：荔枝冻最好还是用浅盒，不然挖到后面全是糖浆。",
|
||||
"[2026-04-11 16:57] 顾澄：海盐脆片如果换成细砂糖版应该会更稳一点。",
|
||||
"[2026-04-11 17:21] 迟雨：百香果软糖配温水挺好，但跟奶酥一起就太重口了。",
|
||||
"[2026-04-11 17:49] 沈砚：我还是投原味苏打一票，夜里拿着方便。",
|
||||
"[2026-04-11 18:12] 许窈：那下轮夜宵我就不带太黏的点心了，省得桌上全是糖粉。",
|
||||
"[2026-04-11 18:43] 顾澄：焦糖海盐脆片和柠檬薄片暂时领先，看起来大家还是更偏爱不容易腻口的那种。",
|
||||
"[2026-04-11 19:18] 迟雨：百香果软糖至少还在前三，不算完全出局。",
|
||||
"[2026-04-11 20:06] 沈砚：乌龙奶酥还是留到白天吧，晚上收拾起来太麻烦。",
|
||||
"[2026-04-11 21:46] 许窈：行，那这轮夜宵就先定海盐脆片、柠檬薄片和百香果软糖，奶酥和荔枝冻留到白天再慢慢吃。"
|
||||
],
|
||||
"expected_memory_targets": [
|
||||
"荔枝冻",
|
||||
"焦糖海盐脆片",
|
||||
"乌龙奶酥",
|
||||
"百香果软糖"
|
||||
]
|
||||
}
|
||||
],
|
||||
"chat_history_records": [
|
||||
{
|
||||
"record_id": 930001,
|
||||
"theme": "北塔夹层药箱补位",
|
||||
"summary": "群里重新确认北塔夹层药箱固定放铜牌备用钥匙、银色保温毯、薄荷膏和黑色丁腈手套，临时取用后必须在群里报备；沈砚再次提到值夜时怕冷白荧光灯直照，会把小风扇调到三档。",
|
||||
"participants": [
|
||||
"Mai",
|
||||
"沈砚",
|
||||
"贺岚",
|
||||
"迟雨",
|
||||
"顾澄"
|
||||
],
|
||||
"start_time": 1774095120.0,
|
||||
"end_time": 1774098180.0,
|
||||
"original_text": "这段北塔夹层药箱的原始聊天记录围绕夜班抽屉补位展开。沈砚先从冷白荧光灯晃眼、自己值夜时会把小风扇调到三档说起，顺手发现铜牌备用钥匙还在，但银色保温毯和薄荷膏的位置已经乱了。贺岚补充黑色丁腈手套在演练后没补齐，顾澄提到旧清单已经与现实摆放不完全一致。Mai 把公用物资重新收束为铜牌备用钥匙、银色保温毯、薄荷膏和黑色丁腈手套四样，要求任何临时取用都在群里补报用途与归还时间，并且明确个人饮料与私人物件不进入抽屉。大家随后又讨论保温毯折法、钥匙居中、手套补回、拍照留档和夜间可见性，让整段群聊既有稳定事实，也有具体流程决议。"
|
||||
},
|
||||
{
|
||||
"record_id": 930002,
|
||||
"theme": "雨棚短波电台返潮排查",
|
||||
"summary": "雨棚那台短波电台返潮后，群里确认灰色防潮箱、黑色盘线和硅胶包的处理顺序需要重排：先擦干机身、松开盘线、替换硅胶包，再按新顺序归箱。",
|
||||
"participants": [
|
||||
"Mai",
|
||||
"顾澄",
|
||||
"贺岚",
|
||||
"迟雨",
|
||||
"沈砚"
|
||||
],
|
||||
"start_time": 1774275240.0,
|
||||
"end_time": 1774278360.0,
|
||||
"original_text": "这段设备维护聊天从短波电台的潮噪开始，但真正讨论的核心很快扩展到灰色防潮箱、黑色盘线、备用听筒、吸水毡和硅胶包如何重新归位。顾澄一开始以为是电台本体故障，贺岚判断更像是防潮箱内环境失效，迟雨进一步指出箱底冷凝水和发软的硅胶包才是主因。Mai 将整个排查流程拆成擦干机身、摊开盘线、检查吸水毡、整包更换硅胶包、最后再按新顺序回箱几步。沈砚还补充了反光、冷风和合盖时机等环境细节。整段记录兼有设备名称、存放规则和操作顺序，能很好地区分“只是提到短波电台”和“真正形成可检索的返潮处理规则”。"
|
||||
},
|
||||
{
|
||||
"record_id": 930003,
|
||||
"theme": "晨雾采样蓝盖保温箱排序",
|
||||
"summary": "晨雾采样结束后，群里把蓝盖保温箱的新布局固定为冷敷袋最下层、编号试剂架居中、玻璃记号笔与封口贴放右槽，样本管最后扣入，避免交接时拿错顺序。",
|
||||
"participants": [
|
||||
"Mai",
|
||||
"迟雨",
|
||||
"顾澄",
|
||||
"贺岚",
|
||||
"沈砚"
|
||||
],
|
||||
"start_time": 1774475280.0,
|
||||
"end_time": 1774478520.0,
|
||||
"original_text": "这段凌晨交接聊天聚焦蓝盖保温箱的内部顺序。大家从冷敷袋压住编号试剂架、玻璃记号笔滑到底层、样本管外壁水汽太重这些小问题谈起，最后把冷敷袋、编号试剂架、玻璃记号笔、封口贴和样本管的排布重新钉死。Mai 明确禁止谁忙就谁随手塞回去的旧习惯，迟雨把位次关系写到箱盖内侧并准备拍照存档，顾澄和贺岚则分别补充了封口贴、吸水纸和标签保护的细节。沈砚提到冷白灯反光会影响编号辨认，这又为后续人物画像提供了背景噪声。整段记录很长，而且既像物资整理又像样本交接，适合检验系统是否能从高重叠词背景里抓住真正的主结论。"
|
||||
},
|
||||
{
|
||||
"record_id": 930004,
|
||||
"theme": "西廊风口观测准备",
|
||||
"summary": "群里把西廊风口观测前准备固定成三步：先校准气压计，再换湿度夹板新纸，最后确认风向丝带、备用夹子和温梨汤的位置；迟雨负责开场前复核气压计。",
|
||||
"participants": [
|
||||
"Mai",
|
||||
"迟雨",
|
||||
"沈砚",
|
||||
"许窈",
|
||||
"贺岚"
|
||||
],
|
||||
"start_time": 1774779720.0,
|
||||
"end_time": 1774783080.0,
|
||||
"original_text": "西廊风口这一段本身并不短，里面混杂着观测流程、环境噪声和人物习惯。迟雨一直在说气压计校准与湿度夹板换纸，许窈在安顿温梨汤和保温壶，贺岚在整理风向丝带、备用夹子和细线，而沈砚则不断从灯光太刺眼、小风扇可能扰动读数这些角度插话。Mai 最终把真正需要记住的主流程收束成三步，并提醒公用桌上不要混入私人物件。由于这一段既有流程结论，也有大量背景描述，它特别适合验证 profile 是否还能稳定抓住迟雨和沈砚各自的稳定事实，而不是被场景语句淹没。"
|
||||
},
|
||||
{
|
||||
"record_id": 930005,
|
||||
"theme": "绘图室天窗白耳鸮闯入",
|
||||
"summary": "白耳鸮闯进绘图室后，群里确认应急顺序是先稳住象牙描图纸，再关天窗，最后把琥珀夹子与黄铜镇纸按原位归回；绘图室天窗边长桌的固定摆放规则保持不变。",
|
||||
"participants": [
|
||||
"Mai",
|
||||
"许窈",
|
||||
"顾澄",
|
||||
"贺岚",
|
||||
"沈砚"
|
||||
],
|
||||
"start_time": 1775133060.0,
|
||||
"end_time": 1775136540.0,
|
||||
"original_text": "这段聊天从白耳鸮扑进绘图室开始，话题里先后出现天窗缝、折梯、象牙描图纸、琥珀夹子、黄铜镇纸、玻璃尺、灯光角度和风扇停开等多个细节。许窈和顾澄一开始更关心压纸和找夹子，贺岚负责天窗和折梯，沈砚提醒冷白灯会刺激鸟并且让纸面反光，Mai 则把整件事稳定成先稳纸、再关窗、最后恢复桌面固定摆放的规则。因为里面既有明显的事件经过，又有不少可能误导检索的场景词，所以很适合当作自然语言 episode query 的压力样本。"
|
||||
}
|
||||
],
|
||||
"person_writebacks": [
|
||||
{
|
||||
"person_id": "71aa50b4f4f34bdbb1a7a1c1e56c9011",
|
||||
"person_name": "沈砚",
|
||||
"memory_content": "沈砚值夜时不喜欢冷白荧光灯直照，通常把小风扇固定在三档，并且会自带盐渍梅子气泡水。",
|
||||
"expected_keywords": [
|
||||
"沈砚",
|
||||
"冷白荧光灯",
|
||||
"小风扇",
|
||||
"三档",
|
||||
"盐渍梅子气泡水"
|
||||
]
|
||||
},
|
||||
{
|
||||
"person_id": "82bb79d4680849b5ac7e23a3f6f09122",
|
||||
"person_name": "迟雨",
|
||||
"memory_content": "迟雨每次西廊观测前都会先校准气压计并核对湿度夹板，而且只喝温梨汤。",
|
||||
"expected_keywords": [
|
||||
"迟雨",
|
||||
"气压计",
|
||||
"湿度夹板",
|
||||
"校准",
|
||||
"温梨汤"
|
||||
]
|
||||
},
|
||||
{
|
||||
"person_id": "93cb73d41251472a8d229f1202df9333",
|
||||
"person_name": "贺岚",
|
||||
"memory_content": "贺岚平时看管北塔夹层药箱和铜牌备用钥匙，取用后会把黑色丁腈手套和银色保温毯补回原位。",
|
||||
"expected_keywords": [
|
||||
"贺岚",
|
||||
"北塔夹层药箱",
|
||||
"铜牌备用钥匙",
|
||||
"黑色丁腈手套",
|
||||
"银色保温毯"
|
||||
]
|
||||
}
|
||||
],
|
||||
"search_cases": [
|
||||
{
|
||||
"query": "北塔夹层 铜牌备用钥匙 银色保温毯 薄荷膏",
|
||||
"expected_keywords": [
|
||||
"北塔夹层",
|
||||
"铜牌备用钥匙",
|
||||
"银色保温毯",
|
||||
"薄荷膏"
|
||||
],
|
||||
"minimum_keyword_hits": 2
|
||||
},
|
||||
{
|
||||
"query": "短波电台 灰色防潮箱 黑色盘线 硅胶包",
|
||||
"expected_keywords": [
|
||||
"短波电台",
|
||||
"灰色防潮箱",
|
||||
"黑色盘线",
|
||||
"硅胶包"
|
||||
],
|
||||
"minimum_keyword_hits": 2
|
||||
},
|
||||
{
|
||||
"query": "蓝盖保温箱 编号试剂架 冷敷袋 玻璃记号笔",
|
||||
"expected_keywords": [
|
||||
"蓝盖保温箱",
|
||||
"编号试剂架",
|
||||
"冷敷袋",
|
||||
"玻璃记号笔"
|
||||
],
|
||||
"minimum_keyword_hits": 2
|
||||
},
|
||||
{
|
||||
"query": "白耳鸮 琥珀夹子 象牙描图纸 黄铜镇纸",
|
||||
"expected_keywords": [
|
||||
"白耳鸮",
|
||||
"琥珀夹子",
|
||||
"象牙描图纸",
|
||||
"黄铜镇纸"
|
||||
],
|
||||
"minimum_keyword_hits": 2
|
||||
}
|
||||
],
|
||||
"time_cases": [
|
||||
{
|
||||
"query": "铜牌钥匙 保温毯",
|
||||
"time_expression": "2026/03/21",
|
||||
"expected_keywords": [
|
||||
"铜牌备用钥匙",
|
||||
"银色保温毯",
|
||||
"薄荷膏"
|
||||
]
|
||||
},
|
||||
{
|
||||
"query": "气压计 梨汤",
|
||||
"time_expression": "2026/03/29",
|
||||
"expected_keywords": [
|
||||
"气压计",
|
||||
"温梨汤",
|
||||
"西廊风口"
|
||||
]
|
||||
},
|
||||
{
|
||||
"query": "白耳鸮 描图纸",
|
||||
"time_expression": "2026/04/02",
|
||||
"expected_keywords": [
|
||||
"白耳鸮",
|
||||
"象牙描图纸",
|
||||
"琥珀夹子"
|
||||
]
|
||||
}
|
||||
],
|
||||
"episode_cases": [
|
||||
{
|
||||
"query": "那回半夜说北塔夹层抽屉里只剩铜牌钥匙，还得把银色保温毯和薄荷膏补回去的那整段经过",
|
||||
"expected_keywords": [
|
||||
"北塔夹层",
|
||||
"铜牌备用钥匙",
|
||||
"银色保温毯",
|
||||
"薄荷膏"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"query": "雨棚底下那回先把电台擦干，又讨论灰色防潮箱和黑色盘线怎么归位的来龙去脉",
|
||||
"expected_keywords": [
|
||||
"短波电台",
|
||||
"灰色防潮箱",
|
||||
"黑色盘线",
|
||||
"硅胶包"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"query": "黎明前重新排蓝盖保温箱、编号试剂架和冷敷袋顺序的那次交接",
|
||||
"expected_keywords": [
|
||||
"蓝盖保温箱",
|
||||
"编号试剂架",
|
||||
"冷敷袋",
|
||||
"玻璃记号笔"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"query": "天窗边那只白耳鸮冲进来，把琥珀夹子和象牙描图纸搅乱之后大家怎么收拾的",
|
||||
"expected_keywords": [
|
||||
"白耳鸮",
|
||||
"琥珀夹子",
|
||||
"象牙描图纸",
|
||||
"黄铜镇纸"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
}
|
||||
],
|
||||
"knowledge_fetcher_cases": [
|
||||
{
|
||||
"query": "群里后来把北塔夹层药箱里固定放哪些东西？",
|
||||
"expected_keywords": [
|
||||
"北塔夹层药箱",
|
||||
"铜牌备用钥匙",
|
||||
"银色保温毯",
|
||||
"薄荷膏"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"query": "谁每次观测前都会先校准气压计，而且只喝温梨汤？",
|
||||
"expected_keywords": [
|
||||
"迟雨",
|
||||
"气压计",
|
||||
"校准",
|
||||
"温梨汤"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"query": "那次白耳鸮闯进绘图室后，大家把哪些东西重新压好固定？",
|
||||
"expected_keywords": [
|
||||
"白耳鸮",
|
||||
"琥珀夹子",
|
||||
"象牙描图纸",
|
||||
"黄铜镇纸"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
}
|
||||
],
|
||||
"profile_cases": [
|
||||
{
|
||||
"person_id": "71aa50b4f4f34bdbb1a7a1c1e56c9011",
|
||||
"expected_keywords": [
|
||||
"沈砚",
|
||||
"冷白荧光灯",
|
||||
"三档",
|
||||
"盐渍梅子气泡水"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"person_id": "82bb79d4680849b5ac7e23a3f6f09122",
|
||||
"expected_keywords": [
|
||||
"迟雨",
|
||||
"气压计",
|
||||
"湿度夹板",
|
||||
"温梨汤"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
},
|
||||
{
|
||||
"person_id": "93cb73d41251472a8d229f1202df9333",
|
||||
"expected_keywords": [
|
||||
"贺岚",
|
||||
"北塔夹层药箱",
|
||||
"铜牌备用钥匙",
|
||||
"黑色丁腈手套"
|
||||
],
|
||||
"minimum_keyword_recall": 0.75
|
||||
}
|
||||
],
|
||||
"negative_control_cases": [
|
||||
{
|
||||
"query": "焦糖海盐脆片 荔枝冻 乌龙奶酥 百香果软糖",
|
||||
"source_batch_id": "dessert_idle_april_04_negative",
|
||||
"expected_behavior": "should_return_no_hits_if_only_positive_batches_are_ingested",
|
||||
"reason": "当前设计要求没有 Bot 发言的群聊批次不应进入长期记忆总结主路径；这条负样本还故意拉长并保留丰富词面，避免仅靠“太短”通过。"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"meta": {
|
||||
"scenario_id": "long_novel_memory_benchmark_v1",
|
||||
"description": "长篇叙事检索基准，围绕北塔木梯、蓝漆铁盒与海潮图展开，验证导入、摘要写回、人物写回、episode 与工具检索链路。",
|
||||
"designed_for": "A_memorix long novel benchmark",
|
||||
"quantitative_targets": {
|
||||
"search_accuracy_at_1": 0.35,
|
||||
"search_recall_at_5": 0.6,
|
||||
"search_keyword_recall_at_5": 0.8,
|
||||
"writeback_success_rate": 0.66,
|
||||
"knowledge_fetcher_success_rate": 0.66,
|
||||
"profile_success_rate": 0.66,
|
||||
"tool_modes_success_rate": 0.75
|
||||
}
|
||||
},
|
||||
"session": {
|
||||
"session_id": "private:novel:beita",
|
||||
"platform": "qq",
|
||||
"user_id": "10001",
|
||||
"group_id": "",
|
||||
"display_name": "北塔潮痕"
|
||||
},
|
||||
"import_payload": {
|
||||
"paragraphs": [
|
||||
{
|
||||
"content": "北塔木梯第四阶后面的夹层里藏着一个蓝漆铁盒，铁盒里压着卷起的海潮图，图角写着旧码头会在大潮夜开启暗门。",
|
||||
"source": "fixture:long_novel_memory_benchmark",
|
||||
"knowledge_type": "narrative",
|
||||
"entities": ["北塔木梯", "蓝漆铁盒", "海潮图", "旧码头", "暗门"]
|
||||
},
|
||||
{
|
||||
"content": "沈砚秋每次登北塔前都会先用银壳指南针校正方向，再把海潮图折成三折塞回蓝漆铁盒；她提醒顾回，真正藏铁盒的位置不是第七阶，而是北塔木梯第四阶。",
|
||||
"source": "fixture:long_novel_memory_benchmark",
|
||||
"knowledge_type": "narrative",
|
||||
"entities": ["沈砚秋", "银壳指南针", "海潮图", "蓝漆铁盒", "顾回", "北塔木梯第四阶"]
|
||||
},
|
||||
{
|
||||
"content": "顾回在北塔木梯旁的潮痕墙上看见一幅褪色海潮图摹本，他据此确认蓝漆铁盒里的红线标记指向旧码头东侧的石闸门。",
|
||||
"source": "fixture:long_novel_memory_benchmark",
|
||||
"knowledge_type": "narrative",
|
||||
"entities": ["顾回", "北塔木梯", "海潮图", "蓝漆铁盒", "旧码头", "石闸门"]
|
||||
},
|
||||
{
|
||||
"content": "他们后来把蓝漆铁盒重新藏回北塔木梯第四阶后面，只留下海潮图的摹本带去旧码头，避免有人提前发现暗门的位置。",
|
||||
"source": "fixture:long_novel_memory_benchmark",
|
||||
"knowledge_type": "narrative",
|
||||
"entities": ["蓝漆铁盒", "北塔木梯第四阶", "海潮图", "旧码头", "暗门"]
|
||||
}
|
||||
],
|
||||
"relations": [
|
||||
{
|
||||
"subject": "蓝漆铁盒",
|
||||
"predicate": "藏在",
|
||||
"object": "北塔木梯第四阶后面"
|
||||
},
|
||||
{
|
||||
"subject": "蓝漆铁盒",
|
||||
"predicate": "装有",
|
||||
"object": "海潮图"
|
||||
},
|
||||
{
|
||||
"subject": "海潮图",
|
||||
"predicate": "指向",
|
||||
"object": "旧码头东侧石闸门"
|
||||
},
|
||||
{
|
||||
"subject": "沈砚秋",
|
||||
"predicate": "使用",
|
||||
"object": "银壳指南针"
|
||||
}
|
||||
]
|
||||
},
|
||||
"chat_history_records": [
|
||||
{
|
||||
"record_id": 930001,
|
||||
"theme": "北塔木梯与蓝漆铁盒",
|
||||
"summary": "沈砚秋和顾回确认蓝漆铁盒一直藏在北塔木梯第四阶后面，铁盒里压着海潮图，海潮图上的红线指向旧码头东侧石闸门；他们最终把铁盒放回原位，只带走摹本继续查暗门。",
|
||||
"participants": ["Mai", "沈砚秋", "顾回"],
|
||||
"start_time": 1775041200.0,
|
||||
"end_time": 1775043600.0,
|
||||
"original_text": "[2026-04-01 18:00] 沈砚秋：我又去摸了一遍北塔木梯，蓝漆铁盒还在第四阶后面的夹层里。\n[2026-04-01 18:02] 顾回：里面那张海潮图的红线还是指着旧码头东侧石闸门，看来暗门位置没有变。\n[2026-04-01 18:04] Mai：你们把铁盒放回去了吗？\n[2026-04-01 18:05] 沈砚秋：放回去了，只带了海潮图摹本，免得别人发现第四阶的夹层。\n[2026-04-01 18:07] 顾回：银壳指南针也重新校过了，等夜潮退下去我们再去旧码头。 "
|
||||
}
|
||||
],
|
||||
"person_writebacks": [
|
||||
{
|
||||
"person_id": "person-shen-yanqiu",
|
||||
"person_name": "沈砚秋",
|
||||
"memory_content": "沈砚秋习惯用银壳指南针校正方向，确认北塔木梯第四阶后的蓝漆铁盒和海潮图位置后才继续行动。",
|
||||
"expected_keywords": ["沈砚秋", "银壳指南针", "蓝漆铁盒", "海潮图"]
|
||||
}
|
||||
],
|
||||
"search_cases": [
|
||||
{
|
||||
"query": "蓝漆铁盒 北塔木梯 海潮图 在哪里",
|
||||
"expected_keywords": ["蓝漆铁盒", "北塔木梯", "海潮图"],
|
||||
"minimum_keyword_hits": 2
|
||||
}
|
||||
],
|
||||
"knowledge_fetcher_cases": [
|
||||
{
|
||||
"query": "他们后来是在什么位置找到蓝漆铁盒和海潮图的？",
|
||||
"expected_keywords": ["蓝漆铁盒", "北塔木梯第四阶", "海潮图"],
|
||||
"minimum_keyword_recall": 0.67
|
||||
}
|
||||
],
|
||||
"profile_cases": [
|
||||
{
|
||||
"person_id": "person-shen-yanqiu",
|
||||
"expected_keywords": ["沈砚秋", "银壳指南针", "海潮图"],
|
||||
"minimum_keyword_recall": 0.67
|
||||
}
|
||||
],
|
||||
"episode_cases": [
|
||||
{
|
||||
"query": "那次他们在北塔木梯寻找蓝漆铁盒和海潮图的经过",
|
||||
"expected_keywords": ["蓝漆铁盒", "北塔木梯第四阶", "海潮图"],
|
||||
"minimum_keyword_recall": 0.67
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"meta": {
|
||||
"scenario_id": "private_alice_weekend",
|
||||
"description": "私聊周末回忆场景，用于验证导入/检索/画像/episode 全链路"
|
||||
},
|
||||
"session": {
|
||||
"session_id": "qq_private_10001",
|
||||
"platform": "qq",
|
||||
"user_id": "10001",
|
||||
"group_id": "",
|
||||
"display_name": "小爱"
|
||||
},
|
||||
"import_payload": {
|
||||
"paragraphs": [
|
||||
{
|
||||
"content": "周末我们在北塔木梯旁找到一个蓝漆铁盒，盒里夹着海潮图。",
|
||||
"source": "fixture:private_alice_weekend",
|
||||
"knowledge_type": "narrative",
|
||||
"entities": [
|
||||
"小爱",
|
||||
"蓝漆铁盒",
|
||||
"北塔木梯",
|
||||
"海潮图"
|
||||
]
|
||||
},
|
||||
{
|
||||
"content": "小爱说蓝漆铁盒是她外公留下的，她会把海潮图带去修复。",
|
||||
"source": "fixture:private_alice_weekend",
|
||||
"knowledge_type": "factual",
|
||||
"entities": [
|
||||
"小爱",
|
||||
"蓝漆铁盒",
|
||||
"海潮图"
|
||||
]
|
||||
}
|
||||
],
|
||||
"relations": [
|
||||
{
|
||||
"subject": "小爱",
|
||||
"predicate": "发现",
|
||||
"object": "蓝漆铁盒"
|
||||
},
|
||||
{
|
||||
"subject": "蓝漆铁盒",
|
||||
"predicate": "内含",
|
||||
"object": "海潮图"
|
||||
},
|
||||
{
|
||||
"subject": "小爱",
|
||||
"predicate": "提到地点",
|
||||
"object": "北塔木梯"
|
||||
}
|
||||
]
|
||||
},
|
||||
"chat_history_record": {
|
||||
"record_id": 900001,
|
||||
"theme": "周末北塔发现",
|
||||
"summary": "小爱在北塔木梯旁找到蓝漆铁盒，铁盒里有海潮图，并计划修复。",
|
||||
"participants": [
|
||||
"小爱",
|
||||
"Mai"
|
||||
],
|
||||
"start_time": 1732903200.0,
|
||||
"end_time": 1732904100.0,
|
||||
"original_text": "周六傍晚，小爱说她在北塔木梯边发现了一个蓝漆铁盒，里面夹着海潮图，想下周拿去修复。"
|
||||
},
|
||||
"person": {
|
||||
"person_id": "7ee9e14d602520af84e57b8665f8e14d",
|
||||
"person_name": "小爱"
|
||||
},
|
||||
"person_fact": {
|
||||
"memory_content": "小爱熟悉北塔木梯路线，知道蓝漆铁盒里夹着海潮图。"
|
||||
},
|
||||
"search_queries": {
|
||||
"direct": "蓝漆铁盒 北塔木梯 海潮图",
|
||||
"knowledge_fetcher": "小爱周末提到的铁盒和地图是什么"
|
||||
},
|
||||
"expectations": {
|
||||
"search_keywords": [
|
||||
"蓝漆铁盒",
|
||||
"北塔木梯",
|
||||
"海潮图"
|
||||
],
|
||||
"profile_keywords": [
|
||||
"小爱",
|
||||
"蓝漆铁盒"
|
||||
],
|
||||
"episode_source": "chat_summary:qq_private_10001"
|
||||
}
|
||||
}
|
||||
166
pytests/A_memorix_test/test_embedding_dimension_control.py
Normal file
166
pytests/A_memorix_test/test_embedding_dimension_control.py
Normal file
@@ -0,0 +1,166 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from A_memorix.core.embedding import api_adapter as api_adapter_module
|
||||
from A_memorix.core.embedding.api_adapter import EmbeddingAPIAdapter
|
||||
from A_memorix.core.utils.runtime_self_check import run_embedding_runtime_self_check
|
||||
|
||||
|
||||
class _FakeEmbeddingClient:
|
||||
def __init__(self, *, natural_dimension: int = 12) -> None:
|
||||
self.natural_dimension = int(natural_dimension)
|
||||
self.requests = []
|
||||
|
||||
async def get_embedding(self, request):
|
||||
self.requests.append(request)
|
||||
requested_dimension = request.extra_params.get("dimensions")
|
||||
if requested_dimension is None:
|
||||
requested_dimension = request.extra_params.get("output_dimensionality")
|
||||
dimension = int(requested_dimension or self.natural_dimension)
|
||||
return SimpleNamespace(embedding=[1.0] * dimension)
|
||||
|
||||
|
||||
def _build_adapter(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
client_type: str,
|
||||
configured_dimension: int = 1024,
|
||||
effective_dimension: int | None = None,
|
||||
model_extra_params: dict | None = None,
|
||||
):
|
||||
adapter = EmbeddingAPIAdapter(default_dimension=configured_dimension)
|
||||
if effective_dimension is not None:
|
||||
adapter._dimension = int(effective_dimension)
|
||||
adapter._dimension_detected = True
|
||||
|
||||
fake_client = _FakeEmbeddingClient()
|
||||
model_info = SimpleNamespace(
|
||||
name="embedding-model",
|
||||
api_provider="provider-1",
|
||||
model_identifier="embedding-model-id",
|
||||
extra_params=dict(model_extra_params or {}),
|
||||
)
|
||||
provider = SimpleNamespace(name="provider-1", client_type=client_type)
|
||||
|
||||
monkeypatch.setattr(adapter, "_resolve_candidate_model_names", lambda: ["embedding-model"])
|
||||
monkeypatch.setattr(adapter, "_find_model_info", lambda model_name: model_info)
|
||||
monkeypatch.setattr(adapter, "_find_provider", lambda provider_name: provider)
|
||||
monkeypatch.setattr(
|
||||
api_adapter_module.client_registry,
|
||||
"get_client_class_instance",
|
||||
lambda api_provider, force_new=True: fake_client,
|
||||
)
|
||||
return adapter, fake_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_encode_uses_canonical_dimension_for_openai_provider(monkeypatch):
|
||||
adapter, fake_client = _build_adapter(
|
||||
monkeypatch,
|
||||
client_type="openai",
|
||||
configured_dimension=1024,
|
||||
effective_dimension=1024,
|
||||
model_extra_params={"task_type": "SEMANTIC_SIMILARITY"},
|
||||
)
|
||||
|
||||
embedding = await adapter.encode("北塔木梯")
|
||||
|
||||
request = fake_client.requests[-1]
|
||||
assert request.extra_params["dimensions"] == 1024
|
||||
assert "output_dimensionality" not in request.extra_params
|
||||
assert request.extra_params["task_type"] == "SEMANTIC_SIMILARITY"
|
||||
assert embedding.shape == (1024,)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_encode_explicit_dimension_override_wins(monkeypatch):
|
||||
adapter, fake_client = _build_adapter(
|
||||
monkeypatch,
|
||||
client_type="openai",
|
||||
configured_dimension=1024,
|
||||
effective_dimension=1024,
|
||||
)
|
||||
|
||||
embedding = await adapter.encode("海潮图", dimensions=256)
|
||||
|
||||
request = fake_client.requests[-1]
|
||||
assert request.extra_params["dimensions"] == 256
|
||||
assert "output_dimensionality" not in request.extra_params
|
||||
assert embedding.shape == (256,)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_encode_maps_dimension_to_gemini_output_dimensionality(monkeypatch):
|
||||
adapter, fake_client = _build_adapter(
|
||||
monkeypatch,
|
||||
client_type="gemini",
|
||||
configured_dimension=1024,
|
||||
effective_dimension=768,
|
||||
)
|
||||
|
||||
embedding = await adapter.encode("广播站")
|
||||
|
||||
request = fake_client.requests[-1]
|
||||
assert request.extra_params["output_dimensionality"] == 768
|
||||
assert "dimensions" not in request.extra_params
|
||||
assert embedding.shape == (768,)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_encode_does_not_force_dimension_for_unsupported_provider(monkeypatch):
|
||||
adapter, fake_client = _build_adapter(
|
||||
monkeypatch,
|
||||
client_type="custom",
|
||||
configured_dimension=1024,
|
||||
effective_dimension=640,
|
||||
model_extra_params={
|
||||
"dimensions": 999,
|
||||
"output_dimensionality": 888,
|
||||
"custom_flag": "keep-me",
|
||||
},
|
||||
)
|
||||
|
||||
embedding = await adapter.encode("蓝漆铁盒")
|
||||
|
||||
request = fake_client.requests[-1]
|
||||
assert "dimensions" not in request.extra_params
|
||||
assert "output_dimensionality" not in request.extra_params
|
||||
assert request.extra_params["custom_flag"] == "keep-me"
|
||||
assert embedding.shape == (fake_client.natural_dimension,)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_self_check_reports_requested_dimension_without_explicit_override():
|
||||
class _FakeEmbeddingManager:
|
||||
def __init__(self) -> None:
|
||||
self.detected_dimension = 384
|
||||
self.encode_calls = []
|
||||
|
||||
async def _detect_dimension(self) -> int:
|
||||
return self.detected_dimension
|
||||
|
||||
def get_requested_dimension(self) -> int:
|
||||
return self.detected_dimension
|
||||
|
||||
async def encode(self, text):
|
||||
self.encode_calls.append(text)
|
||||
return np.ones(self.detected_dimension, dtype=np.float32)
|
||||
|
||||
manager = _FakeEmbeddingManager()
|
||||
|
||||
report = await run_embedding_runtime_self_check(
|
||||
config={"embedding": {"dimension": 1024}},
|
||||
vector_store=SimpleNamespace(dimension=384),
|
||||
embedding_manager=manager,
|
||||
)
|
||||
|
||||
assert report["ok"] is True
|
||||
assert report["configured_dimension"] == 1024
|
||||
assert report["requested_dimension"] == 384
|
||||
assert report["detected_dimension"] == 384
|
||||
assert report["encoded_dimension"] == 384
|
||||
assert manager.encode_calls == ["A_Memorix runtime self check"]
|
||||
@@ -0,0 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DATA_DIR = Path(__file__).parent / "data" / "benchmarks"
|
||||
|
||||
|
||||
def _fixture_files() -> list[Path]:
|
||||
return sorted(DATA_DIR.glob("group_chat_stream_memory_benchmark*.json"))
|
||||
|
||||
|
||||
def _load_fixture(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _assert_fixture_matches_current_design_constraints(dataset: dict) -> None:
|
||||
assert dataset["meta"]["scenario_id"]
|
||||
|
||||
assert dataset["session"]["group_id"]
|
||||
assert dataset["session"]["platform"] == "qq"
|
||||
|
||||
simulated_batches = dataset["simulated_stream_batches"]
|
||||
assert len(simulated_batches) >= 5
|
||||
|
||||
positive_batches = [item for item in simulated_batches if item["bot_participated"]]
|
||||
negative_batches = [item for item in simulated_batches if not item["bot_participated"]]
|
||||
|
||||
assert len(positive_batches) >= 4
|
||||
assert len(negative_batches) >= 1
|
||||
assert any(item["expected_behavior"] == "ignored_by_summarizer_without_bot_message" for item in negative_batches)
|
||||
|
||||
for batch in positive_batches:
|
||||
assert "Mai" in batch["participants"]
|
||||
assert batch["message_count"] >= 10
|
||||
assert len(batch["combined_text"]) >= 300
|
||||
assert batch["start_time"] < batch["end_time"]
|
||||
assert len(batch["expected_memory_targets"]) >= 4
|
||||
|
||||
runtime_streams = dataset["runtime_trigger_streams"]
|
||||
assert len(runtime_streams) >= 2
|
||||
|
||||
runtime_positive = [item for item in runtime_streams if item["bot_participated"]]
|
||||
runtime_negative = [item for item in runtime_streams if not item["bot_participated"]]
|
||||
|
||||
assert len(runtime_positive) >= 1
|
||||
assert len(runtime_negative) >= 1
|
||||
|
||||
for stream in runtime_streams:
|
||||
stream_text = "\n".join(stream["messages"])
|
||||
assert stream["trigger_mode"] == "time_threshold"
|
||||
assert stream["elapsed_since_last_check_hours"] >= 8.0
|
||||
assert stream["message_count"] >= 20
|
||||
assert len(stream["messages"]) == stream["message_count"]
|
||||
assert len(stream_text) >= 1000
|
||||
assert stream["start_time"] < stream["end_time"]
|
||||
|
||||
assert any(item["expected_check_outcome"] == "should_trigger_topic_check_and_pass_bot_gate" for item in runtime_positive)
|
||||
assert any(
|
||||
item["expected_check_outcome"] == "should_trigger_topic_check_but_be_discarded_without_bot_message"
|
||||
for item in runtime_negative
|
||||
)
|
||||
|
||||
records = dataset["chat_history_records"]
|
||||
assert len(records) >= 4
|
||||
for record in records:
|
||||
assert "Mai" in record["participants"]
|
||||
assert len(record["summary"]) >= 40
|
||||
assert len(record["original_text"]) >= 200
|
||||
assert record["start_time"] < record["end_time"]
|
||||
|
||||
assert len(dataset["person_writebacks"]) >= 3
|
||||
assert len(dataset["search_cases"]) >= 4
|
||||
assert len(dataset["time_cases"]) >= 3
|
||||
assert len(dataset["episode_cases"]) >= 4
|
||||
assert len(dataset["knowledge_fetcher_cases"]) >= 3
|
||||
assert len(dataset["profile_cases"]) >= 3
|
||||
assert len(dataset["negative_control_cases"]) >= 1
|
||||
|
||||
|
||||
def test_group_chat_stream_fixture_matches_current_design_constraints():
|
||||
files = _fixture_files()
|
||||
assert files, "未找到 group_chat_stream_memory_benchmark*.json fixture"
|
||||
for path in files:
|
||||
_assert_fixture_matches_current_design_constraints(_load_fixture(path))
|
||||
1187
pytests/A_memorix_test/test_group_chat_stream_memory_benchmark.py
Normal file
1187
pytests/A_memorix_test/test_group_chat_stream_memory_benchmark.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -7,7 +7,6 @@ from src.services.memory_service import MemoryHit, MemorySearchResult
|
||||
|
||||
|
||||
def test_knowledge_fetcher_resolves_private_memory_context(monkeypatch):
|
||||
monkeypatch.setattr(knowledge_module, "LLMRequest", lambda *args, **kwargs: object())
|
||||
monkeypatch.setattr(
|
||||
knowledge_module,
|
||||
"_chat_manager",
|
||||
@@ -31,7 +30,6 @@ def test_knowledge_fetcher_resolves_private_memory_context(monkeypatch):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_knowledge_fetcher_memory_get_knowledge_uses_memory_service(monkeypatch):
|
||||
monkeypatch.setattr(knowledge_module, "LLMRequest", lambda *args, **kwargs: object())
|
||||
monkeypatch.setattr(
|
||||
knowledge_module,
|
||||
"_chat_manager",
|
||||
@@ -73,7 +71,6 @@ async def test_knowledge_fetcher_memory_get_knowledge_uses_memory_service(monkey
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_knowledge_fetcher_falls_back_to_chat_scope_when_person_scope_misses(monkeypatch):
|
||||
monkeypatch.setattr(knowledge_module, "LLMRequest", lambda *args, **kwargs: object())
|
||||
monkeypatch.setattr(
|
||||
knowledge_module,
|
||||
"_chat_manager",
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, Dict, List
|
||||
@@ -71,21 +72,11 @@ class _KnownPerson:
|
||||
|
||||
|
||||
class _KernelBackedRuntimeManager:
|
||||
is_running = True
|
||||
|
||||
def __init__(self, kernel: SDKMemoryKernel) -> None:
|
||||
self.kernel = kernel
|
||||
|
||||
async def invoke_plugin(
|
||||
self,
|
||||
*,
|
||||
method: str,
|
||||
plugin_id: str,
|
||||
component_name: str,
|
||||
args: Dict[str, Any] | None,
|
||||
timeout_ms: int,
|
||||
):
|
||||
del method, plugin_id, timeout_ms
|
||||
async def invoke(self, component_name: str, args: Dict[str, Any] | None, *, timeout_ms: int = 30000):
|
||||
del timeout_ms
|
||||
payload = args or {}
|
||||
if component_name == "search_memory":
|
||||
return await self.kernel.search_memory(
|
||||
@@ -299,6 +290,12 @@ async def _evaluate_tool_modes(*, session_id: str, dataset: Dict[str, Any]) -> D
|
||||
search_case = dataset["search_cases"][0]
|
||||
episode_case = dataset["episode_cases"][0]
|
||||
aggregate_case = dataset["knowledge_fetcher_cases"][0]
|
||||
first_record = (dataset.get("chat_history_records") or [{}])[0]
|
||||
reference_ts = first_record.get("end_time") or first_record.get("start_time") or 0
|
||||
if reference_ts:
|
||||
time_expression = datetime.fromtimestamp(float(reference_ts)).strftime("%Y/%m/%d")
|
||||
else:
|
||||
time_expression = "最近7天"
|
||||
tool_cases = [
|
||||
{
|
||||
"name": "search",
|
||||
@@ -318,7 +315,7 @@ async def _evaluate_tool_modes(*, session_id: str, dataset: Dict[str, Any]) -> D
|
||||
"mode": "time",
|
||||
"chat_id": session_id,
|
||||
"limit": 5,
|
||||
"time_expression": "最近7天",
|
||||
"time_expression": time_expression,
|
||||
},
|
||||
"expected_keywords": ["蓝漆铁盒", "北塔木梯"],
|
||||
"minimum_keyword_recall": 0.67,
|
||||
@@ -402,7 +399,6 @@ async def benchmark_env(monkeypatch, tmp_path):
|
||||
return {"ok": True, "message": "ok", "encoded_dimension": 32}
|
||||
|
||||
monkeypatch.setattr(kernel_module, "run_embedding_runtime_self_check", fake_self_check)
|
||||
monkeypatch.setattr(memory_service_module, "get_plugin_runtime_manager", None)
|
||||
monkeypatch.setattr(summarizer_module, "_chat_manager", fake_chat_manager)
|
||||
monkeypatch.setattr(knowledge_module, "_chat_manager", fake_chat_manager)
|
||||
monkeypatch.setattr(person_info_module, "_chat_manager", fake_chat_manager)
|
||||
@@ -424,7 +420,7 @@ async def benchmark_env(monkeypatch, tmp_path):
|
||||
},
|
||||
)
|
||||
manager = _KernelBackedRuntimeManager(kernel)
|
||||
monkeypatch.setattr(memory_service_module, "get_plugin_runtime_manager", lambda: manager)
|
||||
monkeypatch.setattr(memory_service_module, "a_memorix_host_service", manager)
|
||||
|
||||
await kernel.initialize()
|
||||
try:
|
||||
|
||||
@@ -10,7 +10,7 @@ import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from A_memorix.core.runtime.sdk_memory_kernel import SDKMemoryKernel
|
||||
from pytests.test_long_novel_memory_benchmark import (
|
||||
from pytests.A_memorix_test.test_long_novel_memory_benchmark import (
|
||||
_evaluate_episode_admin_query,
|
||||
_evaluate_episode_generation,
|
||||
_evaluate_episode_search_mode,
|
||||
@@ -58,7 +58,6 @@ async def benchmark_live_env(monkeypatch, tmp_path):
|
||||
registry = {item["person_name"]: item["person_id"] for item in dataset["person_writebacks"]}
|
||||
reverse_registry = {value: key for key, value in registry.items()}
|
||||
|
||||
monkeypatch.setattr(memory_service_module, "get_plugin_runtime_manager", None)
|
||||
monkeypatch.setattr(summarizer_module, "_chat_manager", fake_chat_manager)
|
||||
monkeypatch.setattr(knowledge_module, "_chat_manager", fake_chat_manager)
|
||||
monkeypatch.setattr(person_info_module, "_chat_manager", fake_chat_manager)
|
||||
@@ -80,7 +79,7 @@ async def benchmark_live_env(monkeypatch, tmp_path):
|
||||
},
|
||||
)
|
||||
manager = _KernelBackedRuntimeManager(kernel)
|
||||
monkeypatch.setattr(memory_service_module, "get_plugin_runtime_manager", lambda: manager)
|
||||
monkeypatch.setattr(memory_service_module, "a_memorix_host_service", manager)
|
||||
|
||||
await kernel.initialize()
|
||||
try:
|
||||
|
||||
@@ -73,7 +73,7 @@ async def test_search_respects_filter_by_default(monkeypatch):
|
||||
{
|
||||
"query": "mai",
|
||||
"limit": 5,
|
||||
"mode": "hybrid",
|
||||
"mode": "search",
|
||||
"chat_id": "stream-1",
|
||||
"person_id": "person-1",
|
||||
"time_start": None,
|
||||
|
||||
@@ -64,7 +64,7 @@ def test_init_all_tools_registers_long_term_memory_tool():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_long_term_memory_search_mode_maps_to_hybrid(monkeypatch):
|
||||
async def test_query_long_term_memory_search_mode_keeps_search(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
async def fake_search(query, **kwargs):
|
||||
@@ -83,7 +83,7 @@ async def test_query_long_term_memory_search_mode_maps_to_hybrid(monkeypatch):
|
||||
"query": "Alice 喜欢什么",
|
||||
"kwargs": {
|
||||
"limit": 5,
|
||||
"mode": "hybrid",
|
||||
"mode": "search",
|
||||
"chat_id": "stream-1",
|
||||
"person_id": "person-1",
|
||||
"time_start": None,
|
||||
|
||||
@@ -59,21 +59,11 @@ class _FakeEmbeddingAdapter:
|
||||
|
||||
|
||||
class _KernelBackedRuntimeManager:
|
||||
is_running = True
|
||||
|
||||
def __init__(self, kernel: SDKMemoryKernel) -> None:
|
||||
self.kernel = kernel
|
||||
|
||||
async def invoke_plugin(
|
||||
self,
|
||||
*,
|
||||
method: str,
|
||||
plugin_id: str,
|
||||
component_name: str,
|
||||
args: Dict[str, Any] | None,
|
||||
timeout_ms: int,
|
||||
):
|
||||
del method, plugin_id, timeout_ms
|
||||
async def invoke(self, component_name: str, args: Dict[str, Any] | None, *, timeout_ms: int = 30000):
|
||||
del timeout_ms
|
||||
payload = args or {}
|
||||
if component_name == "search_memory":
|
||||
return await self.kernel.search_memory(
|
||||
@@ -132,7 +122,6 @@ async def real_dialogue_env(monkeypatch, tmp_path):
|
||||
return {"ok": True, "message": "ok"}
|
||||
|
||||
monkeypatch.setattr(kernel_module, "run_embedding_runtime_self_check", fake_self_check)
|
||||
monkeypatch.setattr(memory_service_module, "get_plugin_runtime_manager", None)
|
||||
monkeypatch.setattr(summarizer_module, "_chat_manager", fake_chat_manager)
|
||||
monkeypatch.setattr(knowledge_module, "_chat_manager", fake_chat_manager)
|
||||
monkeypatch.setattr(person_info_module, "_chat_manager", fake_chat_manager)
|
||||
@@ -148,7 +137,7 @@ async def real_dialogue_env(monkeypatch, tmp_path):
|
||||
},
|
||||
)
|
||||
manager = _KernelBackedRuntimeManager(kernel)
|
||||
monkeypatch.setattr(memory_service_module, "get_plugin_runtime_manager", lambda: manager)
|
||||
monkeypatch.setattr(memory_service_module, "a_memorix_host_service", manager)
|
||||
|
||||
await kernel.initialize()
|
||||
try:
|
||||
|
||||
@@ -32,21 +32,11 @@ def _load_dialogue_fixture() -> Dict[str, Any]:
|
||||
|
||||
|
||||
class _KernelBackedRuntimeManager:
|
||||
is_running = True
|
||||
|
||||
def __init__(self, kernel: SDKMemoryKernel) -> None:
|
||||
self.kernel = kernel
|
||||
|
||||
async def invoke_plugin(
|
||||
self,
|
||||
*,
|
||||
method: str,
|
||||
plugin_id: str,
|
||||
component_name: str,
|
||||
args: Dict[str, Any] | None,
|
||||
timeout_ms: int,
|
||||
):
|
||||
del method, plugin_id, timeout_ms
|
||||
async def invoke(self, component_name: str, args: Dict[str, Any] | None, *, timeout_ms: int = 30000):
|
||||
del timeout_ms
|
||||
payload = args or {}
|
||||
if component_name == "search_memory":
|
||||
return await self.kernel.search_memory(
|
||||
@@ -100,7 +90,6 @@ async def live_dialogue_env(monkeypatch, tmp_path):
|
||||
get_session_name=lambda session_id: session_cfg["display_name"] if session_id == session.session_id else session_id,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(memory_service_module, "get_plugin_runtime_manager", None)
|
||||
monkeypatch.setattr(summarizer_module, "_chat_manager", fake_chat_manager)
|
||||
monkeypatch.setattr(knowledge_module, "_chat_manager", fake_chat_manager)
|
||||
monkeypatch.setattr(person_info_module, "_chat_manager", fake_chat_manager)
|
||||
@@ -116,7 +105,7 @@ async def live_dialogue_env(monkeypatch, tmp_path):
|
||||
},
|
||||
)
|
||||
manager = _KernelBackedRuntimeManager(kernel)
|
||||
monkeypatch.setattr(memory_service_module, "get_plugin_runtime_manager", lambda: manager)
|
||||
monkeypatch.setattr(memory_service_module, "a_memorix_host_service", manager)
|
||||
|
||||
await kernel.initialize()
|
||||
try:
|
||||
|
||||
@@ -3,4 +3,8 @@ from pathlib import Path
|
||||
|
||||
# Add project root to Python path so src imports work
|
||||
project_root = Path(__file__).parent.parent.absolute()
|
||||
sys.path.insert(0, str(project_root))
|
||||
src_root = project_root / "src"
|
||||
if str(src_root) not in sys.path:
|
||||
sys.path.insert(0, str(src_root))
|
||||
if str(project_root) not in sys.path:
|
||||
sys.path.insert(1, str(project_root))
|
||||
|
||||
@@ -5,14 +5,15 @@ import pytest
|
||||
from src.services.memory_service import MemorySearchResult
|
||||
from src.webui.dependencies import require_auth
|
||||
from src.webui.routers import memory as memory_router_module
|
||||
from src.webui.routers.memory import compat_router, router
|
||||
from src.webui.routers.memory import compat_router
|
||||
from src.webui.routes import router as main_router
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client() -> TestClient:
|
||||
app = FastAPI()
|
||||
app.dependency_overrides[require_auth] = lambda: "ok"
|
||||
app.include_router(router)
|
||||
app.include_router(main_router)
|
||||
app.include_router(compat_router)
|
||||
return TestClient(app)
|
||||
|
||||
@@ -20,7 +21,24 @@ def client() -> TestClient:
|
||||
def test_webui_memory_graph_route(client: TestClient, monkeypatch):
|
||||
async def fake_graph_admin(*, action: str, **kwargs):
|
||||
assert action == "get_graph"
|
||||
return {"success": True, "nodes": [], "edges": [], "total_nodes": 0, "limit": kwargs.get("limit")}
|
||||
return {
|
||||
"success": True,
|
||||
"nodes": [],
|
||||
"edges": [
|
||||
{
|
||||
"source": "alice",
|
||||
"target": "map",
|
||||
"weight": 1.5,
|
||||
"relation_hashes": ["rel-1"],
|
||||
"predicates": ["持有"],
|
||||
"relation_count": 1,
|
||||
"evidence_count": 2,
|
||||
"label": "持有",
|
||||
}
|
||||
],
|
||||
"total_nodes": 0,
|
||||
"limit": kwargs.get("limit"),
|
||||
}
|
||||
|
||||
monkeypatch.setattr(memory_router_module.memory_service, "graph_admin", fake_graph_admin)
|
||||
|
||||
@@ -29,6 +47,97 @@ def test_webui_memory_graph_route(client: TestClient, monkeypatch):
|
||||
assert response.status_code == 200
|
||||
assert response.json()["success"] is True
|
||||
assert response.json()["limit"] == 77
|
||||
assert response.json()["edges"][0]["predicates"] == ["持有"]
|
||||
assert response.json()["edges"][0]["relation_count"] == 1
|
||||
assert response.json()["edges"][0]["evidence_count"] == 2
|
||||
|
||||
|
||||
def test_webui_memory_graph_node_detail_route(client: TestClient, monkeypatch):
|
||||
async def fake_graph_admin(*, action: str, **kwargs):
|
||||
assert action == "node_detail"
|
||||
assert kwargs["node_id"] == "Alice"
|
||||
return {
|
||||
"success": True,
|
||||
"node": {"id": "Alice", "type": "entity", "content": "Alice", "appearance_count": 3},
|
||||
"relations": [{"hash": "rel-1", "subject": "Alice", "predicate": "持有", "object": "Map", "text": "Alice 持有 Map", "confidence": 0.9, "paragraph_count": 1, "paragraph_hashes": ["p-1"], "source_paragraph": "p-1"}],
|
||||
"paragraphs": [{"hash": "p-1", "content": "Alice 拿着地图。", "preview": "Alice 拿着地图。", "source": "demo", "entity_count": 2, "relation_count": 1, "entities": ["Alice", "Map"], "relations": ["Alice 持有 Map"]}],
|
||||
"evidence_graph": {
|
||||
"nodes": [{"id": "entity:Alice", "type": "entity", "content": "Alice"}],
|
||||
"edges": [],
|
||||
"focus_entities": ["Alice"],
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(memory_router_module.memory_service, "graph_admin", fake_graph_admin)
|
||||
|
||||
response = client.get("/api/webui/memory/graph/node-detail", params={"node_id": "Alice"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["node"]["id"] == "Alice"
|
||||
assert response.json()["relations"][0]["predicate"] == "持有"
|
||||
assert response.json()["evidence_graph"]["focus_entities"] == ["Alice"]
|
||||
|
||||
|
||||
def test_webui_memory_graph_node_detail_route_returns_404(client: TestClient, monkeypatch):
|
||||
async def fake_graph_admin(*, action: str, **kwargs):
|
||||
assert action == "node_detail"
|
||||
return {"success": False, "error": "未找到节点: Missing"}
|
||||
|
||||
monkeypatch.setattr(memory_router_module.memory_service, "graph_admin", fake_graph_admin)
|
||||
|
||||
response = client.get("/api/webui/memory/graph/node-detail", params={"node_id": "Missing"})
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.json()["detail"] == "未找到节点: Missing"
|
||||
|
||||
|
||||
def test_webui_memory_graph_edge_detail_route(client: TestClient, monkeypatch):
|
||||
async def fake_graph_admin(*, action: str, **kwargs):
|
||||
assert action == "edge_detail"
|
||||
assert kwargs["source"] == "Alice"
|
||||
assert kwargs["target"] == "Map"
|
||||
return {
|
||||
"success": True,
|
||||
"edge": {
|
||||
"source": "Alice",
|
||||
"target": "Map",
|
||||
"weight": 1.5,
|
||||
"relation_hashes": ["rel-1"],
|
||||
"predicates": ["持有"],
|
||||
"relation_count": 1,
|
||||
"evidence_count": 1,
|
||||
"label": "持有",
|
||||
},
|
||||
"relations": [{"hash": "rel-1", "subject": "Alice", "predicate": "持有", "object": "Map", "text": "Alice 持有 Map", "confidence": 0.9, "paragraph_count": 1, "paragraph_hashes": ["p-1"], "source_paragraph": "p-1"}],
|
||||
"paragraphs": [{"hash": "p-1", "content": "Alice 拿着地图。", "preview": "Alice 拿着地图。", "source": "demo", "entity_count": 2, "relation_count": 1, "entities": ["Alice", "Map"], "relations": ["Alice 持有 Map"]}],
|
||||
"evidence_graph": {
|
||||
"nodes": [{"id": "relation:rel-1", "type": "relation", "content": "Alice 持有 Map"}],
|
||||
"edges": [{"source": "paragraph:p-1", "target": "relation:rel-1", "kind": "supports", "label": "支撑", "weight": 1.0}],
|
||||
"focus_entities": ["Alice", "Map"],
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(memory_router_module.memory_service, "graph_admin", fake_graph_admin)
|
||||
|
||||
response = client.get("/api/webui/memory/graph/edge-detail", params={"source": "Alice", "target": "Map"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["edge"]["predicates"] == ["持有"]
|
||||
assert response.json()["paragraphs"][0]["source"] == "demo"
|
||||
assert response.json()["evidence_graph"]["edges"][0]["kind"] == "supports"
|
||||
|
||||
|
||||
def test_webui_memory_graph_edge_detail_route_returns_404(client: TestClient, monkeypatch):
|
||||
async def fake_graph_admin(*, action: str, **kwargs):
|
||||
assert action == "edge_detail"
|
||||
return {"success": False, "error": "未找到边: Alice -> Missing"}
|
||||
|
||||
monkeypatch.setattr(memory_router_module.memory_service, "graph_admin", fake_graph_admin)
|
||||
|
||||
response = client.get("/api/webui/memory/graph/edge-detail", params={"source": "Alice", "target": "Missing"})
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.json()["detail"] == "未找到边: Alice -> Missing"
|
||||
|
||||
|
||||
def test_compat_aggregate_route(client: TestClient, monkeypatch):
|
||||
@@ -42,7 +151,13 @@ def test_compat_aggregate_route(client: TestClient, monkeypatch):
|
||||
response = client.get("/api/query/aggregate", params={"query": "mai"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"success": True, "summary": "summary:mai", "hits": [], "filtered": False}
|
||||
assert response.json() == {
|
||||
"success": True,
|
||||
"summary": "summary:mai",
|
||||
"hits": [],
|
||||
"filtered": False,
|
||||
"error": "",
|
||||
}
|
||||
|
||||
|
||||
def test_auto_save_routes(client: TestClient, monkeypatch):
|
||||
@@ -64,6 +179,80 @@ def test_auto_save_routes(client: TestClient, monkeypatch):
|
||||
assert post_response.json() == {"success": True, "auto_save": False}
|
||||
|
||||
|
||||
def test_memory_config_routes(client: TestClient, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
memory_router_module.a_memorix_host_service,
|
||||
"get_config_schema",
|
||||
lambda: {"layout": {"type": "tabs"}, "sections": {"plugin": {"fields": {}}}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
memory_router_module.a_memorix_host_service,
|
||||
"get_config_path",
|
||||
lambda: memory_router_module.Path("/tmp/config/a_memorix.toml"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
memory_router_module.a_memorix_host_service,
|
||||
"get_config",
|
||||
lambda: {"plugin": {"enabled": True}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
memory_router_module.a_memorix_host_service,
|
||||
"get_raw_config",
|
||||
lambda: "[plugin]\nenabled = true\n",
|
||||
)
|
||||
|
||||
schema_response = client.get("/api/webui/memory/config/schema")
|
||||
config_response = client.get("/api/webui/memory/config")
|
||||
raw_response = client.get("/api/webui/memory/config/raw")
|
||||
|
||||
assert schema_response.status_code == 200
|
||||
assert schema_response.json()["path"] == "/tmp/config/a_memorix.toml"
|
||||
assert schema_response.json()["schema"]["layout"]["type"] == "tabs"
|
||||
|
||||
assert config_response.status_code == 200
|
||||
assert config_response.json() == {
|
||||
"success": True,
|
||||
"config": {"plugin": {"enabled": True}},
|
||||
"path": "/tmp/config/a_memorix.toml",
|
||||
}
|
||||
|
||||
assert raw_response.status_code == 200
|
||||
assert raw_response.json() == {
|
||||
"success": True,
|
||||
"config": "[plugin]\nenabled = true\n",
|
||||
"path": "/tmp/config/a_memorix.toml",
|
||||
}
|
||||
|
||||
|
||||
def test_memory_config_update_routes(client: TestClient, monkeypatch):
|
||||
async def fake_update_config(config):
|
||||
assert config == {"plugin": {"enabled": False}}
|
||||
return {"success": True, "config_path": "config/a_memorix.toml"}
|
||||
|
||||
async def fake_update_raw(raw_config):
|
||||
assert raw_config == "[plugin]\nenabled = false\n"
|
||||
return {"success": True, "config_path": "config/a_memorix.toml"}
|
||||
|
||||
monkeypatch.setattr(memory_router_module.a_memorix_host_service, "update_config", fake_update_config)
|
||||
monkeypatch.setattr(memory_router_module.a_memorix_host_service, "update_raw_config", fake_update_raw)
|
||||
|
||||
config_response = client.put("/api/webui/memory/config", json={"config": {"plugin": {"enabled": False}}})
|
||||
raw_response = client.put("/api/webui/memory/config/raw", json={"config": "[plugin]\nenabled = false\n"})
|
||||
|
||||
assert config_response.status_code == 200
|
||||
assert config_response.json() == {"success": True, "config_path": "config/a_memorix.toml"}
|
||||
|
||||
assert raw_response.status_code == 200
|
||||
assert raw_response.json() == {"success": True, "config_path": "config/a_memorix.toml"}
|
||||
|
||||
|
||||
def test_memory_config_raw_rejects_invalid_toml(client: TestClient):
|
||||
response = client.put("/api/webui/memory/config/raw", json={"config": "[plugin\nenabled = true"})
|
||||
|
||||
assert response.status_code == 400
|
||||
assert "TOML 格式错误" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_recycle_bin_route(client: TestClient, monkeypatch):
|
||||
async def fake_get_recycle_bin(*, limit: int):
|
||||
return {"success": True, "items": [{"hash": "deadbeef"}], "count": 1, "limit": limit}
|
||||
@@ -154,6 +343,85 @@ def test_delete_preview_route(client: TestClient, monkeypatch):
|
||||
assert response.json() == {"success": True, "counts": {"paragraphs": 1}, "dry_run": True}
|
||||
|
||||
|
||||
def test_delete_preview_route_supports_mixed_mode(client: TestClient, monkeypatch):
|
||||
async def fake_delete_admin(*, action: str, **kwargs):
|
||||
assert action == "preview"
|
||||
assert kwargs["mode"] == "mixed"
|
||||
assert kwargs["selector"] == {
|
||||
"entity_hashes": ["entity-1"],
|
||||
"paragraph_hashes": ["p-1"],
|
||||
"relation_hashes": ["rel-1"],
|
||||
"sources": ["demo"],
|
||||
}
|
||||
return {"success": True, "mode": "mixed", "counts": {"entities": 1, "paragraphs": 1, "relations": 1, "sources": 1}}
|
||||
|
||||
monkeypatch.setattr(memory_router_module.memory_service, "delete_admin", fake_delete_admin)
|
||||
|
||||
response = client.post(
|
||||
"/api/webui/memory/delete/preview",
|
||||
json={
|
||||
"mode": "mixed",
|
||||
"selector": {
|
||||
"entity_hashes": ["entity-1"],
|
||||
"paragraph_hashes": ["p-1"],
|
||||
"relation_hashes": ["rel-1"],
|
||||
"sources": ["demo"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["mode"] == "mixed"
|
||||
assert response.json()["counts"]["entities"] == 1
|
||||
|
||||
|
||||
def test_delete_execute_route_supports_mixed_mode(client: TestClient, monkeypatch):
|
||||
async def fake_delete_admin(*, action: str, **kwargs):
|
||||
assert action == "execute"
|
||||
assert kwargs["mode"] == "mixed"
|
||||
assert kwargs["selector"] == {
|
||||
"entity_hashes": ["entity-1"],
|
||||
"paragraph_hashes": ["p-1"],
|
||||
"relation_hashes": ["rel-1"],
|
||||
"sources": ["demo"],
|
||||
}
|
||||
assert kwargs["reason"] == "knowledge_graph_delete_entity"
|
||||
assert kwargs["requested_by"] == "knowledge_graph"
|
||||
return {
|
||||
"success": True,
|
||||
"mode": "mixed",
|
||||
"operation_id": "op-mixed-1",
|
||||
"deleted_count": 4,
|
||||
"deleted_entity_count": 1,
|
||||
"deleted_relation_count": 1,
|
||||
"deleted_paragraph_count": 1,
|
||||
"deleted_source_count": 1,
|
||||
"counts": {"entities": 1, "paragraphs": 1, "relations": 1, "sources": 1},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(memory_router_module.memory_service, "delete_admin", fake_delete_admin)
|
||||
|
||||
response = client.post(
|
||||
"/api/webui/memory/delete/execute",
|
||||
json={
|
||||
"mode": "mixed",
|
||||
"selector": {
|
||||
"entity_hashes": ["entity-1"],
|
||||
"paragraph_hashes": ["p-1"],
|
||||
"relation_hashes": ["rel-1"],
|
||||
"sources": ["demo"],
|
||||
},
|
||||
"reason": "knowledge_graph_delete_entity",
|
||||
"requested_by": "knowledge_graph",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["success"] is True
|
||||
assert response.json()["mode"] == "mixed"
|
||||
assert response.json()["operation_id"] == "op-mixed-1"
|
||||
|
||||
|
||||
def test_episode_process_pending_route(client: TestClient, monkeypatch):
|
||||
async def fake_episode_admin(*, action: str, **kwargs):
|
||||
assert action == "process_pending"
|
||||
@@ -258,6 +526,20 @@ def test_delete_execute_route(client: TestClient, monkeypatch):
|
||||
assert response.json() == {"success": True, "operation_id": "del-1"}
|
||||
|
||||
|
||||
def test_sources_route(client: TestClient, monkeypatch):
|
||||
async def fake_source_admin(*, action: str, **kwargs):
|
||||
assert action == "list"
|
||||
assert kwargs == {}
|
||||
return {"success": True, "items": [{"source": "demo", "paragraph_count": 2}], "count": 1}
|
||||
|
||||
monkeypatch.setattr(memory_router_module.memory_service, "source_admin", fake_source_admin)
|
||||
|
||||
response = client.get("/api/webui/memory/sources")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["items"] == [{"source": "demo", "paragraph_count": 2}]
|
||||
|
||||
|
||||
def test_delete_operation_routes(client: TestClient, monkeypatch):
|
||||
async def fake_delete_admin(*, action: str, **kwargs):
|
||||
if action == "list_operations":
|
||||
|
||||
49
pytests/webui/test_plugin_management_routes.py
Normal file
49
pytests/webui/test_plugin_management_routes.py
Normal file
@@ -0,0 +1,49 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.webui.routers.plugin import management as management_module
|
||||
from src.webui.routers.plugin import support as support_module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(tmp_path, monkeypatch) -> TestClient:
|
||||
plugins_dir = tmp_path / "plugins"
|
||||
plugins_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
demo_dir = plugins_dir / "demo_plugin"
|
||||
demo_dir.mkdir()
|
||||
(demo_dir / "_manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"manifest_version": 2,
|
||||
"id": "test.demo",
|
||||
"name": "Demo Plugin",
|
||||
"version": "1.0.0",
|
||||
"description": "demo plugin",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(management_module, "require_plugin_token", lambda _: "ok")
|
||||
monkeypatch.setattr(support_module, "get_plugins_dir", lambda: plugins_dir)
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(management_module.router, prefix="/api/webui/plugins")
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_installed_plugins_only_scan_plugins_dir_and_exclude_a_memorix(client: TestClient):
|
||||
response = client.get("/api/webui/plugins/installed")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["success"] is True
|
||||
|
||||
ids = [plugin["id"] for plugin in payload["plugins"]]
|
||||
assert ids == ["test.demo"]
|
||||
assert "a-dawn.a-memorix" not in ids
|
||||
assert all("/src/plugins/built_in/" not in plugin["path"] for plugin in payload["plugins"])
|
||||
@@ -830,14 +830,7 @@ run_installation() {
|
||||
echo -e "${RED}克隆MaiCore仓库失败！${RESET}"
|
||||
exit 1
|
||||
}
|
||||
echo -e "${GREEN}初始化MaiCore子模块...${RESET}"
|
||||
# 使用与主仓一致的 GitHub 加速前缀，避免子模块直连 github.com 失败
|
||||
git -C MaiBot config submodule.plugins/A_memorix.url "$GITHUB_REPO/A-Dawn/A_memorix.git"
|
||||
git -C MaiBot submodule sync --recursive
|
||||
git -C MaiBot submodule update --init --recursive || {
|
||||
echo -e "${RED}初始化MaiCore子模块失败！${RESET}"
|
||||
exit 1
|
||||
}
|
||||
echo -e "${GREEN}A_Memorix 已内置到源码，无需初始化子模块。${RESET}"
|
||||
|
||||
echo -e "${GREEN}克隆 maim_message 包仓库...${RESET}"
|
||||
git clone $GITHUB_REPO/MaiM-with-u/maim_message.git || {
|
||||
|
||||
25
scripts/run_a_memorix_webui_backend.py
Normal file
25
scripts/run_a_memorix_webui_backend.py
Normal file
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from src.A_memorix.host_service import a_memorix_host_service
|
||||
from src.webui.webui_server import get_webui_server
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
server = get_webui_server()
|
||||
await a_memorix_host_service.start()
|
||||
try:
|
||||
await server.start()
|
||||
finally:
|
||||
await a_memorix_host_service.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
21
scripts/sync_a_memorix_subtree.sh
Executable file
21
scripts/sync_a_memorix_subtree.sh
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MODE="${1:-pull}"
|
||||
REMOTE_URL="${2:-https://github.com/A-Dawn/A_memorix.git}"
|
||||
BRANCH="${3:-MaiBot_branch}"
|
||||
PREFIX="src/A_memorix"
|
||||
|
||||
case "$MODE" in
|
||||
add)
|
||||
git subtree add --prefix="$PREFIX" "$REMOTE_URL" "$BRANCH" --squash
|
||||
;;
|
||||
pull)
|
||||
git subtree pull --prefix="$PREFIX" "$REMOTE_URL" "$BRANCH" --squash
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 [add|pull] [remote_url] [branch]" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
83
scripts/verify_a_memorix_webui.sh
Executable file
83
scripts/verify_a_memorix_webui.sh
Executable file
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DASHBOARD_ROOT="$REPO_ROOT/dashboard"
|
||||
OUTPUT_DIR="${MAIBOT_UI_SNAPSHOT_DIR:-$REPO_ROOT/tmp/ui-snapshots/a_memorix-electron}"
|
||||
PYTHON_BIN="${MAIBOT_PYTHON_BIN:-$REPO_ROOT/../../.venv/bin/python}"
|
||||
ELECTRON_BIN="${MAIBOT_ELECTRON_BIN:-$DASHBOARD_ROOT/node_modules/electron/dist/Electron.app/Contents/MacOS/Electron}"
|
||||
DRIVER_SCRIPT="$DASHBOARD_ROOT/scripts/a_memorix_electron_validate.cjs"
|
||||
BACKEND_SCRIPT="$REPO_ROOT/scripts/run_a_memorix_webui_backend.py"
|
||||
BACKEND_HOST="${MAIBOT_WEBUI_HOST:-127.0.0.1}"
|
||||
BACKEND_PORT="${MAIBOT_WEBUI_PORT:-8001}"
|
||||
DASHBOARD_HOST="${MAIBOT_DASHBOARD_HOST:-127.0.0.1}"
|
||||
DASHBOARD_PORT="${MAIBOT_DASHBOARD_PORT:-7999}"
|
||||
BACKEND_URL="http://${BACKEND_HOST}:${BACKEND_PORT}"
|
||||
DASHBOARD_URL="http://${DASHBOARD_HOST}:${DASHBOARD_PORT}"
|
||||
REUSE_SERVICES="${MAIBOT_UI_REUSE_SERVICES:-0}"
|
||||
|
||||
BACKEND_PID=""
|
||||
DASHBOARD_PID=""
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
if [[ -n "$DASHBOARD_PID" ]] && kill -0 "$DASHBOARD_PID" >/dev/null 2>&1; then
|
||||
kill "$DASHBOARD_PID" >/dev/null 2>&1 || true
|
||||
wait "$DASHBOARD_PID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ -n "$BACKEND_PID" ]] && kill -0 "$BACKEND_PID" >/dev/null 2>&1; then
|
||||
kill "$BACKEND_PID" >/dev/null 2>&1 || true
|
||||
wait "$BACKEND_PID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
wait_for_url() {
|
||||
local url="$1"
|
||||
local label="$2"
|
||||
local timeout="${3:-60}"
|
||||
local started_at
|
||||
started_at="$(date +%s)"
|
||||
while true; do
|
||||
if env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY NO_PROXY=127.0.0.1,localhost \
|
||||
curl -fsS "$url" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if (( "$(date +%s)" - started_at >= timeout )); then
|
||||
echo "Timed out waiting for ${label}: ${url}" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
if [[ "$REUSE_SERVICES" != "1" ]]; then
|
||||
if ! env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY NO_PROXY=127.0.0.1,localhost \
|
||||
curl -fsS "${BACKEND_URL}/api/webui/health" >/dev/null 2>&1; then
|
||||
(
|
||||
cd "$REPO_ROOT"
|
||||
WEBUI_HOST="$BACKEND_HOST" WEBUI_PORT="$BACKEND_PORT" "$PYTHON_BIN" "$BACKEND_SCRIPT"
|
||||
) >"$OUTPUT_DIR/backend.log" 2>&1 &
|
||||
BACKEND_PID="$!"
|
||||
wait_for_url "${BACKEND_URL}/api/webui/health" "MaiBot WebUI backend" 120
|
||||
fi
|
||||
|
||||
if ! env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY NO_PROXY=127.0.0.1,localhost \
|
||||
curl -fsS "${DASHBOARD_URL}/auth" >/dev/null 2>&1; then
|
||||
(
|
||||
cd "$DASHBOARD_ROOT"
|
||||
npm run dev -- --host "$DASHBOARD_HOST" --port "$DASHBOARD_PORT"
|
||||
) >"$OUTPUT_DIR/dashboard.log" 2>&1 &
|
||||
DASHBOARD_PID="$!"
|
||||
wait_for_url "${DASHBOARD_URL}/auth" "dashboard dev server" 120
|
||||
fi
|
||||
fi
|
||||
|
||||
env -u ELECTRON_RUN_AS_NODE \
|
||||
MAIBOT_DASHBOARD_URL="$DASHBOARD_URL" \
|
||||
MAIBOT_UI_SNAPSHOT_DIR="$OUTPUT_DIR" \
|
||||
"$ELECTRON_BIN" "$DRIVER_SCRIPT"
|
||||
2
src/A_memorix/.gitattributes
vendored
Normal file
2
src/A_memorix/.gitattributes
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
245
src/A_memorix/.gitignore
vendored
Normal file
245
src/A_memorix/.gitignore
vendored
Normal file
@@ -0,0 +1,245 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
#uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||
.pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
# Ruff stuff:
|
||||
.ruff_cache/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
# Cursor
|
||||
# Cursor is an AI-powered code editor.`.cursorignore` specifies files/directories to
|
||||
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
||||
# refer to https://docs.cursor.com/context/ignore-files
|
||||
.cursorignore
|
||||
.cursorindexingignore
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
*.egg-info/
|
||||
|
||||
# Data & Storage (Privacy & Runtime)
|
||||
data/
|
||||
logs/
|
||||
|
||||
# Deprecated / Cleanup (Avoid uploading junk)
|
||||
deprecated/
|
||||
|
||||
# OS / System
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
ehthumbs.db
|
||||
|
||||
# IDE settings
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# Temporary Verification Scripts
|
||||
verify_*.py
|
||||
config.toml
|
||||
|
||||
# Test Artifacts & Generated Files
|
||||
MagicMock/
|
||||
benchmark_output.txt
|
||||
e2e_debug.log
|
||||
e2e_error.log
|
||||
full_diff.txt
|
||||
|
||||
# Large Test Data Files
|
||||
机娘导论-openie.json
|
||||
scripts/机娘导论-openie.json
|
||||
|
||||
# A_memorix recall/tuning generated artifacts
|
||||
artifacts/
|
||||
scripts/run_arc_light_recall_pipeline.py
|
||||
|
||||
# Compressed Data Archives
|
||||
data.zip
|
||||
scripts/full_feature_smoke_test.py
|
||||
ACL2026_DEMO_EVAL.md
|
||||
.probe_write
|
||||
tests/
|
||||
temp_verify_v5_data/metadata/metadata.db
|
||||
sql2/t.db
|
||||
sql2/t.db-journal
|
||||
scripts/test.json
|
||||
scripts/test1.json
|
||||
scripts/test-sample.json
|
||||
USAGE_ARCHITECTURE.md
|
||||
scripts/test_conversion.py
|
||||
scripts/debug_graph_vis.py
|
||||
/.tmp_feature_e2e_real
|
||||
/.tmp_sparse_tests
|
||||
/.tmp_test_probe
|
||||
/.tmp_test_sqlite
|
||||
/.tmp_testdata
|
||||
/scripts/tmp
|
||||
718
src/A_memorix/CHANGELOG.md
Normal file
718
src/A_memorix/CHANGELOG.md
Normal file
@@ -0,0 +1,718 @@
|
||||
# 更新日志 (Changelog)
|
||||
|
||||
## [2.0.0] - 2026-03-18
|
||||
|
||||
本次 `2.0.0` 为架构收敛版本，主线是 **SDK Tool 接口统一**、**管理工具能力补齐**、**元数据 schema 升级到 v8** 与 **文档口径同步到 2.0.0**。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`1.0.1` → `2.0.0`
|
||||
- 元数据 schema：`7` → `8`
|
||||
|
||||
### 🚀 重点能力
|
||||
|
||||
- Tool 接口统一：
|
||||
- `plugin.py` 统一通过 `SDKMemoryKernel` 对外提供 Tool 能力。
|
||||
- 保留基础工具：`search_memory / ingest_summary / ingest_text / get_person_profile / maintain_memory / memory_stats`。
|
||||
- 新增管理工具：`memory_graph_admin / memory_source_admin / memory_episode_admin / memory_profile_admin / memory_runtime_admin / memory_import_admin / memory_tuning_admin / memory_v5_admin / memory_delete_admin`。
|
||||
- 检索与写入治理增强：
|
||||
- 检索/写入链路支持 `respect_filter + user_id/group_id` 的聊天过滤语义。
|
||||
- `maintain_memory` 支持 `freeze` 与 `recycle_bin`，并统一到内核维护流程。
|
||||
- 导入与调优能力收敛：
|
||||
- `memory_import_admin` 提供任务化导入能力（上传、粘贴、扫描、OpenIE、LPMM 转换、时序回填、MaiBot 迁移）。
|
||||
- `memory_tuning_admin` 提供检索调优任务（创建、轮次查看、回滚、apply_best、报告导出）。
|
||||
- V5 与删除运维：
|
||||
- 新增 `memory_v5_admin`（`reinforce/weaken/remember_forever/forget/restore/status`）。
|
||||
- 新增 `memory_delete_admin`（`preview/execute/restore/list/get/purge`），支持操作审计与恢复。
|
||||
|
||||
### 🛠️ 存储与运行时
|
||||
|
||||
- `metadata_store` 升级到 `SCHEMA_VERSION = 8`。
|
||||
- 新增/完善外部引用与运维记录能力（包括 `external_memory_refs`、`memory_v5_operations`、`delete_operations` 相关数据结构）。
|
||||
- `SDKMemoryKernel` 增加统一后台任务编排（自动保存、Episode pending 处理、画像刷新、记忆维护）。
|
||||
|
||||
### 📚 文档同步
|
||||
|
||||
- `README.md`、`QUICK_START.md`、`CONFIG_REFERENCE.md`、`IMPORT_GUIDE.md` 已切换到 `2.0.0` 口径。
|
||||
- 文档主入口统一为 SDK Tool 工作流，不再以旧版 slash 命令作为主说明路径。
|
||||
|
||||
## [1.0.1] - 2026-03-07
|
||||
|
||||
本次 `1.0.1` 为 `1.0.0` 发布后的热修复版本，主线是 **图谱 WebUI 取数稳定性修复**、**大图过滤性能修复** 与 **真实检索调优链路稳定性修复**。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`1.0.0` → `1.0.1`
|
||||
- 配置版本：`4.1.0`（不变）
|
||||
|
||||
### 🛠️ 代码修复
|
||||
|
||||
- 图谱接口稳定性：
|
||||
- 修复 `/api/graph` 在“磁盘已有图文件但运行时尚未装载入内存”场景下返回空图的问题，接口现在会自动补加载持久化图数据。
|
||||
- 修复问题数据集下 WebUI 打开图谱页时看似“没有任何节点”的现象；根因不是图数据消失，而是后端过滤路径过慢。
|
||||
- 图谱过滤性能：
|
||||
- 优化 `/api/graph?exclude_leaf=true` 的叶子过滤逻辑，改为预计算 hub 邻接关系，不再对每个节点反复做高成本边权查询。
|
||||
- 优化 `GraphStore.get_neighbors()` 并补充入邻居访问能力，避免稠密矩阵展开导致的大图性能退化。
|
||||
- 检索调优稳定性：
|
||||
- 修复真实调优任务在构建运行时配置时深拷贝 `plugin.config`，误复制注入的存储实例并触发 `cannot pickle '_thread.RLock' object` 的问题。
|
||||
- 调优评估改为跳过顶层运行时实例键，仅保留纯配置字段后再附加运行时依赖，真实 WebUI 调优任务可正常启动。
|
||||
|
||||
### 📚 文档同步
|
||||
|
||||
- 同步更新 `README.md`、`CHANGELOG.md`、`CONFIG_REFERENCE.md` 与版本元数据（`plugin.py`、`__init__.py`、`_manifest.json`）。
|
||||
- README 新增 `v1.0.1` 修复说明，并补充“调优前先做 runtime self-check”的建议。
|
||||
|
||||
## [1.0.0] - 2026-03-06
|
||||
|
||||
本次 `1.0.0` 为主版本升级，主线是 **运行时架构模块化**、**Episode 情景记忆闭环**、**聚合检索与图召回增强**、**离线迁移 / 运行时自检 / 检索调优中心**。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`0.7.0` → `1.0.0`
|
||||
- 配置版本：`4.1.0`（不变）
|
||||
|
||||
### 🚀 重点能力
|
||||
|
||||
- 运行时重构：
|
||||
- `plugin.py` 大幅瘦身，生命周期、后台任务、请求路由、检索运行时初始化拆分到 `core/runtime/*`。
|
||||
- 配置 schema 抽离到 `core/config/plugin_config_schema.py`，`_manifest.json` 同步扩展新配置项。
|
||||
- 检索与查询增强：
|
||||
- `KnowledgeQueryTool` 拆分为 query mode + orchestrator，新增长 `aggregate` / `episode` 查询模式。
|
||||
- 新增图辅助关系召回、统一 forward/runtime 构建与请求去重桥接。
|
||||
- Episode / 运维能力：
|
||||
- `metadata_store` schema 升级到 `SCHEMA_VERSION = 7`，新增 `episodes` / `episode_paragraphs` / rebuild queue 等结构。
|
||||
- 新增 `release_vnext_migrate.py`、`runtime_self_check.py`、`rebuild_episodes.py` 与 Web 检索调优页 `web/tuning.html`。
|
||||
|
||||
### 📚 文档同步
|
||||
|
||||
- 版本号同步到 `plugin.py`、`__init__.py`、`_manifest.json`、`README.md` 与 `CONFIG_REFERENCE.md`。
|
||||
- 新增 `RELEASE_SUMMARY_1.0.0.md`
|
||||
|
||||
## [0.7.0] - 2026-03-04
|
||||
|
||||
本次 `0.7.0` 为中版本升级，主线是 **关系向量化闭环（写入 + 状态机 + 回填 + 审计）**、**检索/命令链路增强** 与 **导入任务能力补齐**。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`0.6.1` → `0.7.0`
|
||||
- 配置版本：`4.1.0`（不变）
|
||||
|
||||
### 🚀 重点能力
|
||||
|
||||
- 关系向量化闭环：
|
||||
- 新增统一关系写入服务 `RelationWriteService`（metadata 先写、向量后写，失败进入状态机而非回滚主数据）。
|
||||
- `relations` 侧补齐 `vector_state/retry_count/last_error/updated_at` 等状态字段，支持 `none/pending/ready/failed` 统一治理。
|
||||
- 插件新增后台回填循环与统计接口，可持续修复关系向量缺失并暴露覆盖率指标。
|
||||
- 检索与命令链路增强：
|
||||
- 检索主链继续收敛到 `search/time` forward 路由，`legacy` 仅保留兼容别名。
|
||||
- relation 查询规格解析收口，结构化查询与语义回退边界更清晰。
|
||||
- `/query stats` 与 tool stats 补充关系向量化统计输出。
|
||||
- 导入与运维增强：
|
||||
- Web Import 新增 `temporal_backfill` 任务入口与编排处理。
|
||||
- 新增一致性审计与离线回填脚本，支持灰度修复历史数据。
|
||||
|
||||
### 📚 文档同步
|
||||
|
||||
- 同步更新 `README.md`、`CONFIG_REFERENCE.md` 与本日志版本信息。
|
||||
- `README.md` 新增关系向量审计/回填脚本使用说明，并更新 `convert_lpmm.py` 的关系向量重建行为描述。
|
||||
|
||||
## [0.6.1] - 2026-03-03
|
||||
|
||||
本次 `0.6.1` 为热修复小版本，重点修复 WebUI 插件配置接口在 A_Memorix 场景下的 `tomlkit` 节点序列化兼容问题。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`0.6.0` → `0.6.1`
|
||||
- 配置版本：`4.1.0`（不变）
|
||||
|
||||
### 🛠️ 代码修复
|
||||
|
||||
- 新增运行时补丁 `_patch_webui_a_memorix_routes_for_tomlkit_serialization()`：
|
||||
- 仅包裹 `/api/webui/plugins/config/{plugin_id}` 及其 schema 的 `GET` 路由。
|
||||
- 仅在 `plugin_id == "A_Memorix"` 时，将返回中的 `config/schema` 通过 `to_builtin_data` 原生化。
|
||||
- 保持 `/api/webui/config/*` 全局接口行为不变，避免对其他插件或核心配置路径产生副作用。
|
||||
- 在插件初始化时执行该补丁，确保 WebUI 读取插件配置时返回结构可稳定序列化。
|
||||
|
||||
### 📚 文档同步
|
||||
|
||||
- 同步更新 `README.md`、`CONFIG_REFERENCE.md` 与本日志中的版本信息及修复说明。
|
||||
|
||||
## [0.6.0] - 2026-03-02
|
||||
|
||||
本次 `0.6.0` 为中版本升级，主线是 **Web Import 导入中心上线与脚本能力对齐**、**失败重试机制升级**、**删除后 manifest 同步** 与 **导入链路稳定性增强**。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`0.5.1` → `0.6.0`
|
||||
- 配置版本：`4.0.1` → `4.1.0`
|
||||
|
||||
### 🚀 重点能力
|
||||
|
||||
- 新增 Web Import 导入中心（`/import`）：
|
||||
- 上传/粘贴/本地扫描/LPMM OpenIE/LPMM 转换/时序回填/MaiBot 迁移。
|
||||
- 任务/文件/分块三级状态展示，支持取消与失败重试。
|
||||
- 导入文档弹窗读取（远程优先，失败回退本地）。
|
||||
- 失败重试升级为“分块优先 + 文件回退”：
|
||||
- `POST /api/import/tasks/{task_id}/retry_failed` 保持原路径，语义升级。
|
||||
- 支持对 `extracting` 失败分块进行子集重试。
|
||||
- `writing`/JSON 解析失败自动回退为文件级重试。
|
||||
- 删除后 manifest 同步失效：
|
||||
- 覆盖 `/api/source/batch_delete` 与 `/api/source`。
|
||||
- 返回 `manifest_cleanup` 明细，避免误命中去重跳过重导入。
|
||||
|
||||
### 📂 变更文件清单（本次发布）
|
||||
|
||||
新增文件：
|
||||
|
||||
- `core/utils/web_import_manager.py`
|
||||
- `scripts/migrate_maibot_memory.py`
|
||||
- `web/import.html`
|
||||
|
||||
修改文件：
|
||||
|
||||
- `CHANGELOG.md`
|
||||
- `CONFIG_REFERENCE.md`
|
||||
- `IMPORT_GUIDE.md`
|
||||
- `QUICK_START.md`
|
||||
- `README.md`
|
||||
- `__init__.py`
|
||||
- `_manifest.json`
|
||||
- `components/commands/debug_server_command.py`
|
||||
- `core/embedding/api_adapter.py`
|
||||
- `core/storage/graph_store.py`
|
||||
- `core/utils/summary_importer.py`
|
||||
- `plugin.py`
|
||||
- `requirements.txt`
|
||||
- `server.py`
|
||||
- `web/index.html`
|
||||
|
||||
删除文件：
|
||||
|
||||
- 无
|
||||
|
||||
### 📚 文档同步
|
||||
|
||||
- 同步更新 `README.md`、`QUICK_START.md`、`CONFIG_REFERENCE.md`、`IMPORT_GUIDE.md` 与本日志。
|
||||
- `IMPORT_GUIDE.md` 新增 “Web Import 导入中心” 专区，统一说明能力范围、状态语义与安全边界。
|
||||
|
||||
## [0.5.1] - 2026-02-23
|
||||
|
||||
本次 `0.5.1` 为热修订小版本，重点修复“随主程序启动的后台任务拉起”“空名单过滤语义”以及“知识抽取模型选择”。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`0.5.0` → `0.5.1`
|
||||
- 配置版本：`4.0.0` → `4.0.1`
|
||||
|
||||
### 🛠️ 代码修复
|
||||
|
||||
- 生命周期接入主程序事件：
|
||||
- 新增 `a_memorix_start_handler`（`ON_START`）调用 `plugin.on_enable()`；
|
||||
- 新增 `a_memorix_stop_handler`（`ON_STOP`）调用 `plugin.on_disable()`；
|
||||
- 解决仅注册插件但未触发生命周期时，定时导入任务不启动的问题。
|
||||
- 聊天过滤空列表策略调整：
|
||||
- `whitelist + []`：全部拒绝；
|
||||
- `blacklist + []`：全部放行。
|
||||
- 知识抽取模型选择逻辑调整（`import_command._select_model`）：
|
||||
- `advanced.extraction_model` 现在支持三种语义：任务名 / 模型名 / `auto`；
|
||||
- `auto` 优先抽取相关任务（`lpmm_entity_extract`、`lpmm_rdf_build` 等），并避免误落到 `embedding`；
|
||||
- 当配置无法识别时输出告警并回退自动选择，提高导入阶段的模型选择可预期性。
|
||||
|
||||
### 📚 文档同步
|
||||
|
||||
- 同步更新 `README.md`、`CONFIG_REFERENCE.md` 与 `CHANGELOG.md`。
|
||||
- 同步修正文档中的空名单过滤行为描述，保持与当前代码一致。
|
||||
|
||||
## [0.5.0] - 2026-02-15
|
||||
|
||||
本次 `0.5.0` 以提交 `66ddc1b98547df3c866b19a3f5dc96e1c8eb7731` 为核心，主线是“人物画像能力上线 + 工具/命令接入 + 版本与文档同步”。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`0.4.0` → `0.5.0`
|
||||
- 配置版本：`3.1.0` → `4.0.0`
|
||||
|
||||
### 🚀 人物画像主特性（核心）
|
||||
|
||||
- 新增人物画像服务：`core/utils/person_profile_service.py`
|
||||
- 支持 `person_id/姓名/别名` 解析。
|
||||
- 聚合图关系证据 + 向量证据，生成画像文本并版本化快照。
|
||||
- 支持手工覆盖（override）与 TTL 快照复用。
|
||||
- 存储层新增人物画像相关表与 API：`core/storage/metadata_store.py`
|
||||
- `person_profile_switches`
|
||||
- `person_profile_snapshots`
|
||||
- `person_profile_active_persons`
|
||||
- `person_profile_overrides`
|
||||
- 新增命令：`/person_profile on|off|status`
|
||||
- 文件：`components/commands/person_profile_command.py`
|
||||
- 作用：按 `stream_id + user_id` 控制自动注入开关（opt-in 模式）。
|
||||
- 查询链路接入人物画像：
|
||||
- `knowledge_query_tool` 新增 `query_type=person`，支持 `person_id` 或别名查询。
|
||||
- `/query person` 与 `/query p` 接入画像查询输出。
|
||||
- 插件生命周期接入画像刷新任务：
|
||||
- 启动/停止统一管理 `person_profile_refresh` 后台任务。
|
||||
- 按活跃窗口自动刷新画像快照。
|
||||
|
||||
### 🛠️ 版本与 schema 同步
|
||||
|
||||
- `plugin.py`：`plugin_version` 更新为 `0.5.0`。
|
||||
- `plugin.py`：`plugin.config_version` 默认值更新为 `4.0.0`。
|
||||
- `config.toml`：`config_version` 基线同步为 `4.0.0`（本地配置文件）。
|
||||
- `__init__.py`：`__version__` 更新为 `0.5.0`。
|
||||
- `_manifest.json`：`version` 更新为 `0.5.0`，`manifest_version` 保持 `1` 。
|
||||
- `manifest_utils.py`：仓库内已兼容更高 manifest 版本；但插件发布默认保持 `manifest_version=1` 。
|
||||
|
||||
### 📚 文档同步
|
||||
|
||||
- 更新 `README.md`、`CONFIG_REFERENCE.md`、`QUICK_START.md`、`USAGE_ARCHITECTURE.md`。
|
||||
- 0.5.0 文档主线改为“人物画像能力 + 版本升级 + 检索链路补充说明”。
|
||||
|
||||
## [0.4.0] - 2026-02-13
|
||||
|
||||
本次 `0.4.0` 版本整合了时序检索增强与后续检索链路增强、稳定性修复和文档同步。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`0.3.3` → `0.4.0`
|
||||
- 配置版本：`3.0.0` → `3.1.0`
|
||||
|
||||
### 🚀 新增
|
||||
|
||||
- 新增 `core/retrieval/sparse_bm25.py`
|
||||
- `SparseBM25Config` / `SparseBM25Index`
|
||||
- FTS5 + BM25 稀疏检索
|
||||
- 支持 `jieba/mixed/char_2gram` 分词与懒加载
|
||||
- 支持 ngram 倒排回退与可选 LIKE 兜底
|
||||
- `DualPathRetriever` 新增 sparse/fusion 配置注入：
|
||||
- embedding 不可用时自动 sparse 回退；
|
||||
- `hybrid` 模式支持向量路 + sparse 路并行候选；
|
||||
- 新增 `FusionConfig` 与 `weighted_rrf` 融合。
|
||||
- `MetadataStore` 新增 FTS/倒排能力：
|
||||
- `paragraphs_fts`、`relations_fts` schema 与回填；
|
||||
- `paragraph_ngrams` 倒排索引与回填；
|
||||
- `fts_search_bm25` / `fts_search_relations_bm25` / `ngram_search_paragraphs`。
|
||||
|
||||
### 🛠️ 组件链路同步
|
||||
|
||||
- `plugin.py`
|
||||
- 新增 `[retrieval.sparse]`、`[retrieval.fusion]` 默认配置；
|
||||
- 初始化并向组件注入 `sparse_index`；
|
||||
- `on_disable` 支持按配置卸载 sparse 连接并释放缓存。
|
||||
- `knowledge_search_action.py` / `query_command.py` / `knowledge_query_tool.py`
|
||||
- 统一接入 sparse/fusion 配置；
|
||||
- 统一注入 `sparse_index`；
|
||||
- `stats` 输出新增 sparse 状态观测。
|
||||
- `requirements.txt`
|
||||
- 新增 `jieba>=0.42.1`（未安装时自动回退 char n-gram）。
|
||||
|
||||
### 🧯 修复与行为调整
|
||||
|
||||
- 修复 `retrieval.ppr_concurrency_limit` 不生效问题：
|
||||
- `DualPathRetriever` 使用配置值初始化 `_ppr_semaphore`，不再被固定值覆盖。
|
||||
- 修复 `char_2gram` 召回失效场景：
|
||||
- FTS miss 时增加 `_fallback_substring_search`，优先 ngram 倒排回退，按配置可选 LIKE 兜底。
|
||||
- 提升可观测性与兼容性：
|
||||
- `get_statistics()` 对向量规模字段兼容读取 `size -> num_vectors -> 0`，避免属性缺失导致异常。
|
||||
- `/query stats` 与 `knowledge_query` 输出包含 sparse 状态（enabled/loaded/tokenizer/doc_count）。
|
||||
|
||||
### 📚 文档
|
||||
|
||||
- `README.md`
|
||||
- 新增检索增强说明、稀疏行为说明、时序回填脚本入口。
|
||||
- `CONFIG_REFERENCE.md`
|
||||
- 补齐 sparse/fusion 参数与触发规则、回退链路、融合实现细节。
|
||||
|
||||
### ⏱️ 时序检索与导入增强
|
||||
|
||||
#### 时序检索能力（分钟级）
|
||||
|
||||
- 新增统一时序查询入口：
|
||||
- `/query time`（别名 `/query t`）
|
||||
- `knowledge_query(query_type=time)`
|
||||
- `knowledge_search(query_type=time|hybrid)`
|
||||
- 查询时间参数统一支持：
|
||||
- `YYYY/MM/DD`
|
||||
- `YYYY/MM/DD HH:mm`
|
||||
- 日期参数自动展开边界：
|
||||
- `from/time_from` -> `00:00`
|
||||
- `to/time_to` -> `23:59`
|
||||
- 查询结果统一回传 `metadata.time_meta`，包含命中时间窗口与命中依据（事件时间或 `created_at` 回退）。
|
||||
|
||||
#### 存储与检索链路
|
||||
|
||||
- 段落存储层支持时序字段：
|
||||
- `event_time`
|
||||
- `event_time_start`
|
||||
- `event_time_end`
|
||||
- `time_granularity`
|
||||
- `time_confidence`
|
||||
- 时序命中采用区间相交逻辑，并遵循“双层时间语义”：
|
||||
- 优先 `event_time/event_time_range`
|
||||
- 缺失时回退 `created_at`（可配置关闭）
|
||||
- 检索排序规则保持：语义优先，时间次排序（新到旧）。
|
||||
- `process_knowledge.py` 新增 `--chat-log` 参数：
|
||||
- 启用后强制使用 `narrative` 策略；
|
||||
- 使用 LLM 对聊天文本进行语义时间抽取（支持相对时间转绝对时间），写入 `event_time/event_time_start/event_time_end`。
|
||||
- 新增 `--chat-reference-time`，用于指定相对时间语义解析的参考时间点。
|
||||
|
||||
#### Schema 与文档同步
|
||||
|
||||
- `_manifest.json` 同步补齐 `retrieval.temporal` 配置 schema。
|
||||
- 配置 schema 版本升级：`config_version` 从 `3.0.0` 提升到 `3.1.0`（`plugin.py` / `config.toml` / 配置文档同步）。
|
||||
- 更新 `README.md`、`CONFIG_REFERENCE.md`、`IMPORT_GUIDE.md`，补充时序检索入口、参数格式与导入时间字段说明。
|
||||
|
||||
## [0.3.3] - 2026-02-11
|
||||
|
||||
本次更新为 **语言一致性补丁版本**，重点收敛知识抽取时的语言漂移问题，要求输出严格贴合原文语言，不做翻译改写。
|
||||
|
||||
### 🛠️ 关键修复
|
||||
|
||||
#### 抽取语言约束
|
||||
|
||||
- `BaseStrategy`:
|
||||
- 移除按 `zh/en/mixed` 分支的语言类型判定逻辑；
|
||||
- 统一为单一约束：抽取值保持原文语言、保留原始术语、禁止翻译。
|
||||
- `NarrativeStrategy` / `FactualStrategy`:
|
||||
- 抽取提示词统一接入上述语言约束；
|
||||
- 明确要求 JSON 键名固定、抽取值遵循原文语言表达。
|
||||
|
||||
#### 导入链路一致性
|
||||
|
||||
- `ImportCommand` 的 LLM 抽取提示词同步强化“优先原文语言、不要翻译”要求，避免脚本与指令导入行为不一致。
|
||||
|
||||
#### 测试与文档
|
||||
|
||||
- 更新 `test_strategies.py`，将语言判定测试调整为统一语言约束测试，并验证提示词中包含禁止翻译约束。
|
||||
- 同步更新注释与文档描述，确保实现与说明一致。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`0.3.2` → `0.3.3`
|
||||
|
||||
## [0.3.2] - 2026-02-11
|
||||
|
||||
本次更新为 **V5 稳定性与兼容性修复版本**，在保持原有业务设计（强化→衰减→冷冻→修剪→回收）的前提下，修复关键链路断裂与误判问题。
|
||||
|
||||
### 🛠️ 关键修复
|
||||
|
||||
#### V5 记忆系统契约与链路
|
||||
|
||||
- `MetadataStore`:
|
||||
- 统一 `mark_relations_inactive(hashes, inactive_since=None)` 调用契约，兼容不同调用方；
|
||||
- 补充 `has_table(table_name)`；
|
||||
- 增加 `restore_relation(hash)` 兼容别名，修复服务层恢复调用断裂；
|
||||
- 修正 `get_entity_gc_candidates` 对孤立节点参数的处理（支持节点名映射到实体 hash）。
|
||||
- `GraphStore`:
|
||||
- 清理 `deactivate_edges` 重复定义并统一返回冻结数量，保证上层日志与断言稳定。
|
||||
- `server.py`:
|
||||
- 修复 `/api/memory/restore` relation 恢复链路；
|
||||
- 清理不可达分支并统一异常路径；
|
||||
- 回收站查询在表检测场景下不再出现错误退空。
|
||||
|
||||
#### 命令与模型选择
|
||||
|
||||
- `/memory` 命令修复 hash 长度判定：以 64 位 `sha256` 为标准，同时兼容历史 32 位输入。
|
||||
- 总结模型选择修复：
|
||||
- 解决 `summarization.model_name = auto` 误命中 `embedding` 问题；
|
||||
- 支持数组与选择器语法（`task:model` / task / model）；
|
||||
- 兼容逗号分隔字符串写法（如 `"utils:model1","utils:model2",replyer`）。
|
||||
|
||||
#### 生命周期与脚本稳定性
|
||||
|
||||
- `plugin.py` 修复后台任务生命周期管理：
|
||||
- 增加 `_scheduled_import_task` / `_auto_save_task` / `_memory_maintenance_task` 句柄；
|
||||
- 避免重复启动；
|
||||
- 插件停用时统一 cancel + await 收敛。
|
||||
- `process_knowledge.py` 修复 tenacity 重试日志级别类型错误（`"WARNING"` → `logging.WARNING`），避免 `KeyError: 'WARNING'`。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`0.3.1` → `0.3.2`
|
||||
|
||||
## [0.3.1] - 2026-02-07
|
||||
|
||||
本次更新为 **稳定性补丁版本**，主要修复脚本导入链路、删除安全性与 LPMM 转换一致性问题。
|
||||
|
||||
### 🛠️ 关键修复
|
||||
|
||||
#### 新增功能
|
||||
|
||||
- 新增 `scripts/convert_lpmm.py`：
|
||||
- 支持将 LPMM 的 `parquet + graph` 数据直接转换为 A_Memorix 存储结构；
|
||||
- 提供 LPMM ID 到 A_Memorix ID 的映射能力，用于图节点/边重写；
|
||||
- 当前实现优先保证检索一致性，关系向量采用安全策略（不直接导入）。
|
||||
|
||||
#### 导入链路
|
||||
|
||||
- 修复 `import_lpmm_json.py` 依赖的 `AutoImporter.import_json_data` 公共入口缺失/不稳定问题，确保外部脚本可稳定调用 JSON 直导入流程。
|
||||
|
||||
#### 删除安全
|
||||
|
||||
- 修复按来源删除时“同一 `(subject, object)` 存在多关系”场景下的误删风险：
|
||||
- `MetadataStore.delete_paragraph_atomic` 新增 `relation_prune_ops`；
|
||||
- 仅在无兄弟关系时才回退删除整条边。
|
||||
- `delete_knowledge.py` 新增保守孤儿实体清理（仅对本次候选实体执行，且需同时满足无段落引用、无关系引用、图无邻居）。
|
||||
- `delete_knowledge.py` 改为读取向量元数据中的真实维度，避免 `dimension=1` 写回污染。
|
||||
|
||||
#### LPMM 转换修复
|
||||
|
||||
- 修复 `convert_lpmm.py` 中向量 ID 与 `MetadataStore` 哈希不一致导致的检索反查失败问题。
|
||||
- 为避免脏召回，转换阶段暂时跳过 `relation.parquet` 的直接向量导入（待关系元数据一一映射能力完善后再恢复）。
|
||||
|
||||
### 🔖 版本信息
|
||||
|
||||
- 插件版本：`0.3.0` → `0.3.1`
|
||||
|
||||
## [0.3.0] - 2026-01-30
|
||||
|
||||
本次更新引入了 **V5 动态记忆系统**，实现了符合生物学特性的记忆衰减、强化与全声明周期管理，并提供了配套的指令与工具。
|
||||
|
||||
### 🧠 记忆系统 (V5)
|
||||
|
||||
#### 核心机制
|
||||
|
||||
- **记忆衰减 (Decay)**: 引入"遗忘曲线"，随时间推移自动降低图谱连接权重。
|
||||
- **访问强化 (Reinforcement)**: "越用越强"，每次检索命中都会刷新记忆活跃度并增强权重。
|
||||
- **生命周期 (Lifecycle)**:
|
||||
- **活跃 (Active)**: 正常参与计算与检索。
|
||||
- **冷冻 (Inactive)**: 权重过低被冻结，不再参与 PPR 计算，但保留语义映射 (Mapping)。
|
||||
- **修剪 (Prune)**: 过期且无保护的冷冻记忆将被移入回收站。
|
||||
- **多重保护**: 支持 **永久锁定 (Pin)** 与 **限时保护 (TTL)**，防止关键记忆被误删。
|
||||
|
||||
#### GraphStore
|
||||
|
||||
- **多关系映射**: 实现 `(u,v) -> Set[Hash]` 映射，确保同一通道下的多重语义关系互不干扰。
|
||||
- **原子化操作**: 新增 `decay`, `deactivate_edges` (软删), `prune_relation_hashes` (硬删) 等原子操作。
|
||||
|
||||
### 🛠️ 指令与工具
|
||||
|
||||
#### Memory Command (`/memory`)
|
||||
|
||||
新增全套记忆维护指令：
|
||||
|
||||
- `/memory status`: 查看记忆系统健康状态（活跃/冷冻/回收站计数）。
|
||||
- `/memory protect <query> [hours]`: 保护记忆。不填时间为永久锁定(Pin)，填时间为临时保护(TTL)。
|
||||
- `/memory reinforce <query>`: 手动强化记忆（绕过冷却时间）。
|
||||
- `/memory restore <hash>`: 从回收站恢复误删记忆（仅当节点存在时重建连接）。
|
||||
|
||||
#### MemoryModifierTool
|
||||
|
||||
- **LLM 能力增强**: 更新工具逻辑，支持 LLM 自主触发 `reinforce`, `weaken`, `remember_forever`, `forget` 操作，并自动映射到 V5 底层逻辑。
|
||||
|
||||
### ⚙️ 配置 (`config.toml`)
|
||||
|
||||
新增 `[memory]` 配置节：
|
||||
|
||||
- `half_life_hours`: 记忆半衰期 (默认 24h)。
|
||||
- `enable_auto_reinforce`: 是否开启检索自动强化。
|
||||
- `prune_threshold`: 冷冻/修剪阈值 (默认 0.1)。
|
||||
|
||||
### 💻 WebUI (v1.4)
|
||||
|
||||
实现了与 V5 记忆系统深度集成的全生命周期管理界面：
|
||||
|
||||
- **可视化增强**:
|
||||
- **冷冻状态**: 非活跃记忆以 **虚线 + 灰色 (Slate-300)** 显示。
|
||||
- **保护状态**: 被 Pin 或保护的记忆带有 **金色 (Amber) 光晕**。
|
||||
- **交互升级**:
|
||||
- **记忆回收站**: 新增 Dock 入口与专用面板，支持浏览删除记录并一键恢复。
|
||||
- **快捷操作**: 边属性面板新增 **强化 (Reinforce)**、**保护 (Protect/Pin)**、**冷冻 (Freeze)** 按钮。
|
||||
- **实时反馈**: 操作后自动刷新图谱布局与样式。
|
||||
|
||||
---
|
||||
|
||||
## [0.2.3] - 2026-01-30
|
||||
|
||||
本次更新主要集中在 **WebUI 交互体验优化** 与 **文档/配置的规范化**。
|
||||
|
||||
### 🎨 WebUI (v1.3)
|
||||
|
||||
#### 加载与同步体验升级
|
||||
|
||||
- **沉浸式加载**: 全新设计的加载遮罩，采用磨砂玻璃背景 (`backdrop-filter`) 与呼吸灯文字动效，提升视觉质感。
|
||||
- **精准状态反馈**: 优化加载逻辑，明确区分“网络同步”与“拓扑计算”阶段，解决数据加载时的闪烁问题。
|
||||
- **新手引导**: 在加载界面新增基础操作提示，降低新用户上手门槛。
|
||||
|
||||
#### 全功能帮助面板
|
||||
|
||||
- **操作指南重构**: 全面翻新“操作指南”面板，新增 Dock 栏功能详解、编辑管理操作及视图配置说明。
|
||||
|
||||
### 🛠️ 工程与规范
|
||||
|
||||
#### plugin.py
|
||||
|
||||
- **配置描述补全**: 修复了 `config_section_descriptions` 中缺失 `summarization`, `schedule`, `filter` 节导致的问题。
|
||||
- **版本号**: `0.2.2` → `0.2.3`
|
||||
|
||||
### ⚙️ 核心与服务
|
||||
|
||||
#### Core
|
||||
|
||||
- **量化逻辑修正**: 修正了 `_scalar_quantize_int8` 函数，确保向量值正确映射到 `[-128, 127]` 区间，提高量化精度。
|
||||
|
||||
#### Server
|
||||
|
||||
- **缓存一致性**: 在执行删除节点/边等修改操作后，显式清除 `_relation_cache`，确保前端获取的关系数据实时更新。
|
||||
|
||||
### 🤖 脚本与数据处理
|
||||
|
||||
#### process_knowledge.py
|
||||
|
||||
- **策略模式重构**: 引入了 `Strategy-Aware` 架构，支持通过 `Narrative` (叙事), `Factual` (事实), `Quote` (引用) 三种策略差异化处理文本(准确说是确认实装)（默认采用 Narrative模式）。
|
||||
- **智能分块纠错**: 新增“分块拯救” (`Chunk Rescue`) 机制，可在长叙事文本中自动识别并提取内嵌的歌词或诗句。
|
||||
|
||||
#### import_lpmm_json.py
|
||||
|
||||
- **LPMM 迁移工具**: 增加了对 LPMM OpenIE JSON 格式的完整支持，能够自动计算 Hash 并迁移实体/关系数据，确保与 A_Memorix 存储格式兼容。
|
||||
|
||||
#### Project
|
||||
|
||||
- **构建清理**: 优化 `.gitignore` 规则
|
||||
|
||||
---
|
||||
|
||||
## [0.2.2] - 2026-01-27
|
||||
|
||||
本次更新专注于提高 **网络请求的鲁棒性**，特别是针对嵌入服务的调用。
|
||||
|
||||
### 🛠️ 稳定性与工程改进
|
||||
|
||||
#### EmbeddingAPI
|
||||
|
||||
- **可配置重试机制**: 新增 `[embedding.retry]` 配置项，允许自定义最大重试次数和等待时间。默认重试次数从 3 次增加到 10 次，以更好应对网络波动。
|
||||
- **配置项**:
|
||||
- `max_attempts`: 最大重试次数 (默认: 10)
|
||||
- `max_wait_seconds`: 最大等待时间 (默认: 30s)
|
||||
- `min_wait_seconds`: 最小等待时间 (默认: 2s)
|
||||
|
||||
#### plugin.py
|
||||
|
||||
- **版本号**: `0.2.1` → `0.2.2`
|
||||
|
||||
---
|
||||
|
||||
## [0.2.1] - 2026-01-26
|
||||
|
||||
本次更新重点在于 **可视化交互的全方位重构** 以及 **底层鲁棒性的进一步增强**。
|
||||
|
||||
### 🎨 可视化与交互重构
|
||||
|
||||
#### WebUI (Glassmorphism)
|
||||
|
||||
- **全新视觉设计**: 采用深色磨砂玻璃 (Glassmorphism) 风格，配合动态渐变背景。
|
||||
- **Dock 菜单栏**: 底部新增 macOS 风格 Dock 栏，聚合所有常用功能。
|
||||
- **显著性视图 (Saliency View)**: 基于 **PageRank** 算法的“信息密度”滑块，支持以此过滤叶子节点，仅展示核心骨干或全量细节。
|
||||
- **功能面板**:
|
||||
- **❓ 操作指南**: 内置交互说明与特性介绍。
|
||||
- **🔍 悬浮搜索**: 支持按拼音/ID 实时过滤节点。
|
||||
- **📂 记忆溯源**: 支持按源文件批量查看和删除记忆数据。
|
||||
- **📖 内容字典**: 列表化展示所有实体与关系，支持排序与筛选。
|
||||
|
||||
### 🛠️ 稳定性与工程改进
|
||||
|
||||
#### EmbeddingAPI
|
||||
|
||||
- **鲁棒性增强**: 引入 `tenacity` 实现指数退避重试机制。
|
||||
- **错误处理**: 失败时返回 `NaN` 向量而非零向量，允许上层逻辑安全跳过。
|
||||
|
||||
#### MetadataStore
|
||||
|
||||
- **自动修复**: 自动检测并修复 `vector_index` 列错位（文件名误存）的历史数据问题。
|
||||
- **数据统计**: 新增 `get_all_sources` 接口支持来源统计。
|
||||
|
||||
#### 脚本与工具
|
||||
|
||||
- **用户体验**: 引入 `rich` 库优化终端输出进度条与状态显示。
|
||||
- **接口开放**: `process_knowledge.py` 新增 `import_json_data` 供外部调用。
|
||||
- **LPMM 迁移**: 新增 `import_lpmm_json.py`，支持导入符合 LPMM 规范的 OpenIE JSON 数据。
|
||||
|
||||
#### plugin.py
|
||||
|
||||
- **版本号**: `0.2.0` → `0.2.1`
|
||||
|
||||
---
|
||||
|
||||
## [0.2.0] - 2026-01-22
|
||||
|
||||
> [!CAUTION]
|
||||
> **不完全兼容变更**：v0.2.0 版本重构了底层存储架构。由于数据结构的重大调整，**旧版本的导入数据无法在新版本中完全无损兼容**。
|
||||
> 虽然部分组件支持自动迁移，但为确保数据一致性和检索质量，**强烈建议在升级后重新使用 `process_knowledge.py` 导入原始数据**。
|
||||
|
||||
本次更新为**重大版本升级**，包含向量存储架构重写、检索逻辑强化及多项稳定性改进。
|
||||
|
||||
### 🚀 核心架构重写
|
||||
|
||||
#### VectorStore: SQ8 量化 + Append-Only 存储
|
||||
|
||||
- **全新存储格式**: 从 `.npy` 迁移至 `vectors.bin`（float16 增量追加）和 `vectors_ids.bin`，大幅减少内存占用。
|
||||
- **原生 SQ8 量化**: 使用 Faiss `IndexScalarQuantizer(QT_8bit)`，替代手动 int8 量化逻辑。
|
||||
- **L2 Normalization 强制化**: 所有向量在存储和检索时统一执行 L2 归一化，确保 Inner Product 等价于 Cosine 相似度。
|
||||
- **Fallback 索引机制**: 新增 `IndexFlatIP` 回退索引，在 SQ8 训练完成前提供检索能力，避免冷启动无结果问题。
|
||||
- **Reservoir Sampling 训练采样**: 使用蓄水池采样收集训练数据（上限 10k），保证小数据集和流式导入场景下的训练样本多样性。
|
||||
- **线程安全**: 新增 `threading.RLock` 保护并发读写操作。
|
||||
- **自动迁移**: 支持从旧版 `.npy` 格式自动迁移至新 `.bin` 格式。
|
||||
|
||||
### ✨ 检索功能增强
|
||||
|
||||
#### KnowledgeQueryTool: 智能回退与多跳路径搜索
|
||||
|
||||
- **Smart Fallback (智能回退)**: 当向量检索置信度低于阈值 (默认 0.6) 时，自动尝试提取查询中的实体进行多跳路径搜索（`_path_search`），增强对间接关系的召回能力。
|
||||
- **结果去重 (`_deduplicate_results`)**: 新增基于内容相似度的安全去重逻辑，防止冗余结果污染 LLM 上下文，同时确保至少保留一条结果。
|
||||
- **语义关系检索 (`_semantic_search_relation`)**: 支持自然语言查询关系（无需 `S|P|O` 格式），内部使用 `REL_ONLY` 策略进行向量检索。
|
||||
- **路径搜索 (`_path_search`)**: 新增 `GraphStore.find_paths` 调用，支持查找两个实体间的间接连接路径（最大深度 3，最多 5 条路径）。
|
||||
- **Clean Output**: LLM 上下文中不再包含原始相似度分数，避免模型偏见。
|
||||
|
||||
#### DualPathRetriever: 并发控制与调试模式
|
||||
|
||||
- **PPR 并发限制 (`ppr_concurrency_limit`)**: 新增 Semaphore 控制 PageRank 计算并发数，防止 CPU 峰值过载。
|
||||
- **Debug 模式**: 新增 `debug` 配置项，启用时打印检索结果原文到日志。
|
||||
- **Entity-Pivot 关系检索**: 优化 `_retrieve_relations_only` 策略，通过检索实体后扩展其关联关系，替代直接检索关系向量。
|
||||
|
||||
### ⚙️ 配置与 Schema 扩展
|
||||
|
||||
#### plugin.py
|
||||
|
||||
- **版本号**: `0.1.3` → `0.2.0`
|
||||
- **默认配置版本**: `config_version` 默认值更新为 `2.0.0`
|
||||
- **新增配置项**:
|
||||
- `retrieval.relation_semantic_fallback` (bool): 是否启用关系查询的语义回退。
|
||||
- `retrieval.relation_fallback_min_score` (float): 语义回退的最小相似度阈值。
|
||||
- **相对路径支持**: `storage.data_dir` 现在支持相对路径（相对于插件目录），默认值改为 `./data`。
|
||||
- **全局实例获取**: 新增 `A_MemorixPlugin.get_global_instance()` 静态方法，供组件可靠获取插件实例。
|
||||
|
||||
#### config.toml / \_manifest.json
|
||||
|
||||
- **新增 `ppr_concurrency_limit`**: 控制 PPR 算法并发数。
|
||||
- **新增训练阈值配置**: `embedding.min_train_threshold` 控制触发 SQ8 训练的最小样本数。
|
||||
|
||||
### 🛠️ 稳定性与工程改进
|
||||
|
||||
#### GraphStore
|
||||
|
||||
- **`find_paths` 方法**: 新增多跳路径查找功能，支持 BFS 搜索指定深度内的实体间路径。
|
||||
- **`find_node` 方法**: 新增大小写不敏感的节点查找。
|
||||
|
||||
#### MetadataStore
|
||||
|
||||
- **Schema 迁移**: 自动添加缺失的 `is_permanent`, `last_accessed`, `access_count` 字段。
|
||||
|
||||
#### 脚本与工具
|
||||
|
||||
- **新增脚本**:
|
||||
- `scripts/diagnose_relations_source.py`: 诊断关系溯源问题。
|
||||
- `scripts/verify_search_robustness.py`: 验证检索鲁棒性。
|
||||
- `scripts/run_stress_test.py`, `stress_test_data.py`: 压力测试套件。
|
||||
- `scripts/migrate_canonicalization.py`, `migrate_paragraph_relations.py`: 数据迁移工具。
|
||||
- **目录整理**: 将大量旧版测试脚本移动至 `deprecated/` 目录。
|
||||
|
||||
### 🗑️ 移除与废弃
|
||||
|
||||
- 废弃 `vectors.npy` 存储格式（自动迁移至 `.bin`）。
|
||||
|
||||
---
|
||||
|
||||
## [0.1.3] - 上一个稳定版本
|
||||
|
||||
- 初始发布，包含基础双路检索功能。
|
||||
- 手动 Int8 向量量化。
|
||||
- 基于 `.npy` 的向量存储。
|
||||
359
src/A_memorix/CONFIG_REFERENCE.md
Normal file
359
src/A_memorix/CONFIG_REFERENCE.md
Normal file
@@ -0,0 +1,359 @@
|
||||
# A_Memorix 配置参考 (v2.0.0)
|
||||
|
||||
本文档对应当前仓库代码（`__version__ = 2.0.0`、`SCHEMA_VERSION = 9`）。
|
||||
|
||||
说明：
|
||||
|
||||
- 本文只覆盖 **当前运行时实际读取** 的配置键。
|
||||
- 默认配置文件路径为 `config/a_memorix.toml`。
|
||||
- 旧版 `/query`、`/memory`、`/visualize` 命令体系相关配置，不再作为主路径说明。
|
||||
- 未配置的键会回退到代码默认值。
|
||||
- 长期记忆控制台已可视化高频常用字段；未展示的长尾高级项仍然有效，请通过“源码模式 / 原始 TOML”编辑。
|
||||
|
||||
## 常用完整配置
|
||||
|
||||
```toml
|
||||
[plugin]
|
||||
enabled = true
|
||||
|
||||
[storage]
|
||||
data_dir = "data/plugins/a-dawn.a-memorix"
|
||||
|
||||
[embedding]
|
||||
model_name = "auto"
|
||||
dimension = 1024
|
||||
batch_size = 32
|
||||
max_concurrent = 5
|
||||
enable_cache = false
|
||||
quantization_type = "int8"
|
||||
|
||||
[embedding.fallback]
|
||||
enabled = true
|
||||
probe_interval_seconds = 180
|
||||
allow_metadata_only_write = true
|
||||
|
||||
[embedding.paragraph_vector_backfill]
|
||||
enabled = true
|
||||
interval_seconds = 60
|
||||
batch_size = 64
|
||||
max_retry = 5
|
||||
|
||||
[retrieval]
|
||||
top_k_paragraphs = 20
|
||||
top_k_relations = 10
|
||||
top_k_final = 10
|
||||
alpha = 0.5
|
||||
enable_ppr = true
|
||||
ppr_alpha = 0.85
|
||||
ppr_timeout_seconds = 1.5
|
||||
ppr_concurrency_limit = 4
|
||||
enable_parallel = true
|
||||
|
||||
[retrieval.sparse]
|
||||
enabled = true
|
||||
backend = "fts5"
|
||||
mode = "auto"
|
||||
tokenizer_mode = "jieba"
|
||||
candidate_k = 80
|
||||
relation_candidate_k = 60
|
||||
|
||||
[threshold]
|
||||
min_threshold = 0.3
|
||||
max_threshold = 0.95
|
||||
percentile = 75.0
|
||||
min_results = 3
|
||||
enable_auto_adjust = true
|
||||
|
||||
[filter]
|
||||
enabled = true
|
||||
mode = "blacklist"
|
||||
chats = []
|
||||
|
||||
[episode]
|
||||
enabled = true
|
||||
generation_enabled = true
|
||||
pending_batch_size = 20
|
||||
pending_max_retry = 3
|
||||
max_paragraphs_per_call = 20
|
||||
max_chars_per_call = 6000
|
||||
source_time_window_hours = 24
|
||||
segmentation_model = "auto"
|
||||
|
||||
[person_profile]
|
||||
enabled = true
|
||||
refresh_interval_minutes = 30
|
||||
active_window_hours = 72
|
||||
max_refresh_per_cycle = 50
|
||||
top_k_evidence = 12
|
||||
|
||||
[memory]
|
||||
enabled = true
|
||||
half_life_hours = 24.0
|
||||
prune_threshold = 0.1
|
||||
freeze_duration_hours = 24.0
|
||||
|
||||
[advanced]
|
||||
enable_auto_save = true
|
||||
auto_save_interval_minutes = 5
|
||||
debug = false
|
||||
|
||||
[web.import]
|
||||
enabled = true
|
||||
max_queue_size = 20
|
||||
max_files_per_task = 200
|
||||
max_file_size_mb = 20
|
||||
max_paste_chars = 200000
|
||||
default_file_concurrency = 2
|
||||
default_chunk_concurrency = 4
|
||||
|
||||
[web.tuning]
|
||||
enabled = true
|
||||
max_queue_size = 8
|
||||
poll_interval_ms = 1200
|
||||
default_intensity = "standard"
|
||||
default_objective = "precision_priority"
|
||||
default_top_k_eval = 20
|
||||
default_sample_size = 24
|
||||
```
|
||||
|
||||
### 可视化与原始 TOML 的分工
|
||||
|
||||
- 长期记忆控制台：适合修改高频项，例如 embedding、检索、Episode、人物画像、导入与调优的常用开关。
|
||||
- 原始 TOML：适合复制整份配置、批量调整参数，或修改未在可视化表单中展示的高级项。
|
||||
- raw-only 高级项仍包括：`retrieval.fusion.*`、`retrieval.search.relation_intent.*`、`retrieval.search.graph_recall.*`、`retrieval.aggregate.*`、`memory.orphan.*`、`advanced.extraction_model`、`web.import.llm_retry.*`、`web.import.path_aliases`、`web.import.convert.*`、`web.tuning.llm_retry.*`、`web.tuning.eval_query_timeout_seconds`。
|
||||
|
||||
## 1. 存储与嵌入
|
||||
|
||||
### `storage`
|
||||
|
||||
- `storage.data_dir` (代码默认 `./data`；当前内置配置推荐 `data/plugins/a-dawn.a-memorix`)
|
||||
: 数据目录。相对路径按 MaiBot 仓库根目录解析。
|
||||
|
||||
### `embedding`
|
||||
|
||||
- `embedding.model_name` (默认 `auto`)
|
||||
: embedding 模型选择。
|
||||
- `embedding.dimension` (默认 `1024`)
|
||||
: 唯一公开的维度控制项。A_Memorix 内部会自动映射为 provider 所需请求字段，并在运行时做真实探测与校验。
|
||||
- `embedding.batch_size` (默认 `32`)
|
||||
- `embedding.max_concurrent` (默认 `5`)
|
||||
- `embedding.enable_cache` (默认 `false`)
|
||||
- `embedding.retry` (默认 `{}`)
|
||||
: embedding 调用重试策略。
|
||||
- `embedding.quantization_type`
|
||||
: 当前主路径仅建议 `int8`。
|
||||
- `embedding.fallback.enabled` (默认 `true`)
|
||||
- `embedding.fallback.probe_interval_seconds` (默认 `180`)
|
||||
- `embedding.fallback.allow_metadata_only_write` (默认 `true`)
|
||||
- `embedding.paragraph_vector_backfill.enabled` (默认 `true`)
|
||||
- `embedding.paragraph_vector_backfill.interval_seconds` (默认 `60`)
|
||||
- `embedding.paragraph_vector_backfill.batch_size` (默认 `64`)
|
||||
- `embedding.paragraph_vector_backfill.max_retry` (默认 `5`)
|
||||
|
||||
## 2. 检索
|
||||
|
||||
### `retrieval` 主键
|
||||
|
||||
- `retrieval.top_k_paragraphs` (默认 `20`)
|
||||
- `retrieval.top_k_relations` (默认 `10`)
|
||||
- `retrieval.top_k_final` (默认 `10`)
|
||||
- `retrieval.alpha` (默认 `0.5`)
|
||||
- `retrieval.enable_ppr` (默认 `true`)
|
||||
- `retrieval.ppr_alpha` (默认 `0.85`)
|
||||
- `retrieval.ppr_timeout_seconds` (默认 `1.5`)
|
||||
- `retrieval.ppr_concurrency_limit` (默认 `4`)
|
||||
- `retrieval.enable_parallel` (默认 `true`)
|
||||
- `retrieval.relation_vectorization.enabled` (默认 `false`)
|
||||
|
||||
### `retrieval.sparse` (`SparseBM25Config`)
|
||||
|
||||
常用键（默认值）：
|
||||
|
||||
- `enabled = true`
|
||||
- `backend = "fts5"`
|
||||
- `lazy_load = true`
|
||||
- `mode = "auto"` (`auto`/`fallback_only`/`hybrid`)
|
||||
- 运行时若 embedding 进入 degraded，会强制按 `fallback_only` 执行读路径（不改用户配置文件）
|
||||
- `tokenizer_mode = "jieba"` (`jieba`/`mixed`/`char_2gram`)
|
||||
- `char_ngram_n = 2`
|
||||
- `candidate_k = 80`
|
||||
- `relation_candidate_k = 60`
|
||||
- `enable_ngram_fallback_index = true`
|
||||
- `enable_relation_sparse_fallback = true`
|
||||
|
||||
### `retrieval.fusion` (`FusionConfig`)
|
||||
|
||||
- `method` (默认 `weighted_rrf`)
|
||||
- `rrf_k` (默认 `60`)
|
||||
- `vector_weight` (默认 `0.7`)
|
||||
- `bm25_weight` (默认 `0.3`)
|
||||
- `normalize_score` (默认 `true`)
|
||||
- `normalize_method` (默认 `minmax`)
|
||||
|
||||
### `retrieval.search.relation_intent` (`RelationIntentConfig`)
|
||||
|
||||
- `enabled` (默认 `true`)
|
||||
- `alpha_override` (默认 `0.35`)
|
||||
- `relation_candidate_multiplier` (默认 `4`)
|
||||
- `preserve_top_relations` (默认 `3`)
|
||||
- `force_relation_sparse` (默认 `true`)
|
||||
- `pair_predicate_rerank_enabled` (默认 `true`)
|
||||
- `pair_predicate_limit` (默认 `3`)
|
||||
|
||||
### `retrieval.search.graph_recall` (`GraphRelationRecallConfig`)
|
||||
|
||||
- `enabled` (默认 `true`)
|
||||
- `candidate_k` (默认 `24`)
|
||||
- `max_hop` (默认 `1`)
|
||||
- `allow_two_hop_pair` (默认 `true`)
|
||||
- `max_paths` (默认 `4`)
|
||||
|
||||
### `retrieval.aggregate`
|
||||
|
||||
- `retrieval.aggregate.rrf_k`
|
||||
- `retrieval.aggregate.weights`
|
||||
|
||||
用于聚合检索阶段混合策略；未配置时走代码默认行为。
|
||||
|
||||
## 3. 阈值过滤
|
||||
|
||||
### `threshold` (`ThresholdConfig`)
|
||||
|
||||
- `threshold.min_threshold` (默认 `0.3`)
|
||||
- `threshold.max_threshold` (默认 `0.95`)
|
||||
- `threshold.percentile` (默认 `75.0`)
|
||||
- `threshold.std_multiplier` (默认 `1.5`)
|
||||
- `threshold.min_results` (默认 `3`)
|
||||
- `threshold.enable_auto_adjust` (默认 `true`)
|
||||
|
||||
## 4. 聊天过滤
|
||||
|
||||
### `filter`
|
||||
|
||||
用于 `respect_filter=true` 场景（检索和写入都支持）。
|
||||
|
||||
```toml
|
||||
[filter]
|
||||
enabled = true
|
||||
mode = "blacklist" # blacklist / whitelist
|
||||
chats = ["group:123", "user:456", "stream:abc"]
|
||||
```
|
||||
|
||||
规则：
|
||||
|
||||
- `blacklist`：命中列表即拒绝
|
||||
- `whitelist`：仅列表内允许
|
||||
- 列表为空时：
|
||||
- `blacklist` => 全允许
|
||||
- `whitelist` => 全拒绝
|
||||
|
||||
## 5. Episode
|
||||
|
||||
### `episode`
|
||||
|
||||
- `episode.enabled` (默认 `true`)
|
||||
- `episode.generation_enabled` (默认 `true`)
|
||||
- `episode.pending_batch_size` (默认 `20`，部分路径默认 `12`)
|
||||
- `episode.pending_max_retry` (默认 `3`)
|
||||
- `episode.max_paragraphs_per_call` (默认 `20`)
|
||||
- `episode.max_chars_per_call` (默认 `6000`)
|
||||
- `episode.source_time_window_hours` (默认 `24`)
|
||||
- `episode.segmentation_model` (默认 `auto`)
|
||||
: 支持 `auto`，也支持填写 `utils/replyer/planner/tool_use` 或具体模型名。
|
||||
|
||||
## 6. 人物画像
|
||||
|
||||
### `person_profile`
|
||||
|
||||
- `person_profile.enabled` (默认 `true`)
|
||||
- `person_profile.refresh_interval_minutes` (默认 `30`)
|
||||
- `person_profile.active_window_hours` (默认 `72`)
|
||||
- `person_profile.max_refresh_per_cycle` (默认 `50`)
|
||||
- `person_profile.top_k_evidence` (默认 `12`)
|
||||
|
||||
## 7. 记忆演化与回收
|
||||
|
||||
### `memory`
|
||||
|
||||
- `memory.enabled` (默认 `true`)
|
||||
- `memory.half_life_hours` (默认 `24.0`)
|
||||
- `memory.base_decay_interval_hours` (默认 `1.0`)
|
||||
- `memory.prune_threshold` (默认 `0.1`)
|
||||
- `memory.freeze_duration_hours` (默认 `24.0`)
|
||||
|
||||
### `memory.orphan`
|
||||
|
||||
- `enable_soft_delete` (默认 `true`)
|
||||
- `entity_retention_days` (默认 `7.0`)
|
||||
- `paragraph_retention_days` (默认 `7.0`)
|
||||
- `sweep_grace_hours` (默认 `24.0`)
|
||||
|
||||
## 8. 高级运行时
|
||||
|
||||
### `advanced`
|
||||
|
||||
- `advanced.enable_auto_save` (默认 `true`)
|
||||
- `advanced.auto_save_interval_minutes` (默认 `5`)
|
||||
- `advanced.debug` (默认 `false`)
|
||||
- `advanced.extraction_model` (默认 `auto`)
|
||||
|
||||
## 9. 导入中心 (`web.import`)
|
||||
|
||||
### 开关与限流
|
||||
|
||||
- `web.import.enabled` (默认 `true`)
|
||||
- `web.import.max_queue_size` (默认 `20`)
|
||||
- `web.import.max_files_per_task` (默认 `200`)
|
||||
- `web.import.max_file_size_mb` (默认 `20`)
|
||||
- `web.import.max_paste_chars` (默认 `200000`)
|
||||
- `web.import.default_file_concurrency` (默认 `2`)
|
||||
- `web.import.default_chunk_concurrency` (默认 `4`)
|
||||
- `web.import.max_file_concurrency` (默认 `6`)
|
||||
- `web.import.max_chunk_concurrency` (默认 `12`)
|
||||
- `web.import.poll_interval_ms` (默认 `1000`)
|
||||
|
||||
### 重试与路径
|
||||
|
||||
- `web.import.llm_retry.max_attempts` (默认 `4`)
|
||||
- `web.import.llm_retry.min_wait_seconds` (默认 `3`)
|
||||
- `web.import.llm_retry.max_wait_seconds` (默认 `40`)
|
||||
- `web.import.llm_retry.backoff_multiplier` (默认 `3`)
|
||||
- `web.import.path_aliases` (默认内置 `raw/lpmm/plugin_data`)
|
||||
|
||||
### 转换阶段
|
||||
|
||||
- `web.import.convert.enable_staging_switch` (默认 `true`)
|
||||
- `web.import.convert.keep_backup_count` (默认 `3`)
|
||||
|
||||
## 10. 调优中心 (`web.tuning`)
|
||||
|
||||
- `web.tuning.enabled` (默认 `true`)
|
||||
- `web.tuning.max_queue_size` (默认 `8`)
|
||||
- `web.tuning.poll_interval_ms` (默认 `1200`)
|
||||
- `web.tuning.eval_query_timeout_seconds` (默认 `10.0`)
|
||||
- `web.tuning.default_intensity` (默认 `standard`，可选 `quick/standard/deep`)
|
||||
- `web.tuning.default_objective` (默认 `precision_priority`，可选 `precision_priority/balanced/recall_priority`)
|
||||
- `web.tuning.default_top_k_eval` (默认 `20`)
|
||||
- `web.tuning.default_sample_size` (默认 `24`)
|
||||
- `web.tuning.llm_retry.max_attempts` (默认 `3`)
|
||||
- `web.tuning.llm_retry.min_wait_seconds` (默认 `2`)
|
||||
- `web.tuning.llm_retry.max_wait_seconds` (默认 `20`)
|
||||
- `web.tuning.llm_retry.backoff_multiplier` (默认 `2`)
|
||||
|
||||
## 11. 兼容性提示
|
||||
|
||||
- 若你从 `1.x` 升级，请优先运行：
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/release_vnext_migrate.py preflight --strict
|
||||
python src/A_memorix/scripts/release_vnext_migrate.py migrate --verify-after
|
||||
python src/A_memorix/scripts/release_vnext_migrate.py verify --strict
|
||||
```
|
||||
|
||||
- 启动前再执行：
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/runtime_self_check.py --json
|
||||
```
|
||||
|
||||
以避免 embedding 维度与向量库不匹配导致运行时异常。
|
||||
335
src/A_memorix/IMPORT_GUIDE.md
Normal file
335
src/A_memorix/IMPORT_GUIDE.md
Normal file
@@ -0,0 +1,335 @@
|
||||
# A_Memorix 导入指南 (v2.0.0)
|
||||
|
||||
本文档对应当前 `2.0.0` 代码路径，覆盖两类导入方式：
|
||||
|
||||
1. 脚本导入（离线批处理）
|
||||
2. `memory_import_admin` 任务导入（在线任务化）
|
||||
|
||||
## 1. 导入前检查
|
||||
|
||||
建议先执行：
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/runtime_self_check.py --json
|
||||
```
|
||||
|
||||
再确认：
|
||||
|
||||
- `storage.data_dir` 路径可写
|
||||
- embedding 配置可用
|
||||
- 若是升级项目，先完成迁移脚本
|
||||
|
||||
## 2. 方式 A：脚本导入（推荐起步）
|
||||
|
||||
## 2.1 原始文本导入
|
||||
|
||||
将 `.txt` 文件放入：
|
||||
|
||||
```text
|
||||
data/plugins/a-dawn.a-memorix/raw/
|
||||
```
|
||||
|
||||
执行：
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/process_knowledge.py
|
||||
```
|
||||
|
||||
常用参数：
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/process_knowledge.py --force
|
||||
python src/A_memorix/scripts/process_knowledge.py --chat-log
|
||||
python src/A_memorix/scripts/process_knowledge.py --chat-log --chat-reference-time "2026/02/12 10:30"
|
||||
```
|
||||
|
||||
## 2.2 OpenIE JSON 导入
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/import_lpmm_json.py <json文件或目录>
|
||||
```
|
||||
|
||||
## 2.3 LPMM 数据转换
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/convert_lpmm.py -i <lpmm数据目录> -o data/plugins/a-dawn.a-memorix
|
||||
```
|
||||
|
||||
## 2.4 历史数据迁移
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/migrate_chat_history.py --help
|
||||
python src/A_memorix/scripts/migrate_maibot_memory.py --help
|
||||
python src/A_memorix/scripts/migrate_person_memory_points.py --help
|
||||
```
|
||||
|
||||
## 2.5 导入后修复与重建
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/backfill_temporal_metadata.py --dry-run
|
||||
python src/A_memorix/scripts/backfill_relation_vectors.py --limit 1000
|
||||
python src/A_memorix/scripts/rebuild_episodes.py --all --wait
|
||||
python src/A_memorix/scripts/audit_vector_consistency.py --json
|
||||
```
|
||||
|
||||
## 3. 方式 B：`memory_import_admin` 任务导入
|
||||
|
||||
`memory_import_admin` 是在线任务化导入入口，适合宿主侧面板或自动化管道。
|
||||
|
||||
### 3.1 常用 action
|
||||
|
||||
- `settings` / `get_settings` / `get_guide`
|
||||
- `path_aliases` / `get_path_aliases`
|
||||
- `resolve_path`
|
||||
- `create_upload`
|
||||
- `create_paste`
|
||||
- `create_raw_scan`
|
||||
- `create_lpmm_openie`
|
||||
- `create_lpmm_convert`
|
||||
- `create_temporal_backfill`
|
||||
- `create_maibot_migration`
|
||||
- `list`
|
||||
- `get`
|
||||
- `chunks` / `get_chunks`
|
||||
- `cancel`
|
||||
- `retry_failed`
|
||||
|
||||
### 3.2 调用示例
|
||||
|
||||
查看运行时设置：
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "memory_import_admin",
|
||||
"arguments": {
|
||||
"action": "settings"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
创建粘贴导入任务：
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "memory_import_admin",
|
||||
"arguments": {
|
||||
"action": "create_paste",
|
||||
"content": "今天完成了检索调优回归。",
|
||||
"input_mode": "plain_text",
|
||||
"source": "manual:worklog"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
查询任务列表：
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "memory_import_admin",
|
||||
"arguments": {
|
||||
"action": "list",
|
||||
"limit": 20
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
查看任务详情：
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "memory_import_admin",
|
||||
"arguments": {
|
||||
"action": "get",
|
||||
"task_id": "<task_id>",
|
||||
"include_chunks": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
重试失败任务：
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "memory_import_admin",
|
||||
"arguments": {
|
||||
"action": "retry_failed",
|
||||
"task_id": "<task_id>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 4. 直接写入 Tool（非任务化）
|
||||
|
||||
若你不需要任务编排，也可以直接调用：
|
||||
|
||||
- `ingest_summary`
|
||||
- `ingest_text`
|
||||
|
||||
示例：
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "ingest_text",
|
||||
"arguments": {
|
||||
"external_id": "note:2026-03-18:001",
|
||||
"source_type": "note",
|
||||
"text": "新的召回阈值方案已通过评审",
|
||||
"chat_id": "group:dev",
|
||||
"tags": ["worklog", "review"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`external_id` 建议全局唯一，用于幂等去重。
|
||||
|
||||
## 5. 时间字段建议
|
||||
|
||||
可用时间字段（按常见优先级）：
|
||||
|
||||
- `timestamp`
|
||||
- `time_start`
|
||||
- `time_end`
|
||||
|
||||
建议：
|
||||
|
||||
- 事件类记录优先写 `time_start/time_end`
|
||||
- 仅有单点时间时写 `timestamp`
|
||||
- 历史数据可先导入，再用 `backfill_temporal_metadata.py` 回填
|
||||
|
||||
## 6. source_type 建议
|
||||
|
||||
常见值：
|
||||
|
||||
- `chat_summary`
|
||||
- `note`
|
||||
- `person_fact`
|
||||
- `lpmm_openie`
|
||||
- `migration`
|
||||
|
||||
建议保持稳定枚举，便于后续按来源治理与重建 Episode。
|
||||
|
||||
## 7. 导入完成后的验证
|
||||
|
||||
建议执行以下顺序：
|
||||
|
||||
1. `memory_stats` 看总量是否增长
|
||||
2. `search_memory`（`mode=search`/`aggregate`）抽检召回
|
||||
3. `memory_episode_admin` 的 `status`/`query` 检查 Episode 生成
|
||||
4. `memory_runtime_admin` 的 `self_check` 再确认运行时健康
|
||||
|
||||
## 8. 常见问题
|
||||
|
||||
### Q1: 导入任务创建成功但无写入
|
||||
|
||||
- 检查聊天过滤配置 `filter`（若 `respect_filter=true` 可能被过滤）
|
||||
- 检查任务详情中的失败原因与分块状态
|
||||
|
||||
### Q2: 任务反复失败
|
||||
|
||||
- 检查 embedding 与 LLM 可用性
|
||||
- 降低并发（`web.import.default_*_concurrency`）
|
||||
- 调整重试参数（`web.import.llm_retry.*`）
|
||||
|
||||
### Q3: 导入后检索效果差
|
||||
|
||||
- 先做 `runtime_self_check`
|
||||
- 检查 `retrieval.sparse` 是否启用
|
||||
- 使用 `memory_tuning_admin` 创建调优任务做参数回归
|
||||
|
||||
## 9. 相关文档
|
||||
|
||||
- [QUICK_START.md](QUICK_START.md)
|
||||
- [CONFIG_REFERENCE.md](CONFIG_REFERENCE.md)
|
||||
- [README.md](README.md)
|
||||
- [CHANGELOG.md](CHANGELOG.md)
|
||||
|
||||
## 10. 附录：策略模式参考
|
||||
|
||||
A_Memorix 导入链路仍然遵循策略模式（Strategy-Aware）。`process_knowledge.py` 会自动识别文本类型，也支持手动指定。
|
||||
|
||||
| 策略类型 | 适用场景 | 核心逻辑 | 自动识别特征 |
|
||||
| :-- | :-- | :-- | :-- |
|
||||
| `Narrative` (叙事) | 小说、同人文、剧本、长篇故事 | 按场景/章节切分，使用滑动窗口；提取事件与角色关系 | `#`、`Chapter`、`***` 等章节标记 |
|
||||
| `Factual` (事实) | 设定集、百科、说明书 | 按语义块切分，保留列表/定义结构；提取 SPO 三元组 | 列表符号、`术语: 解释` |
|
||||
| `Quote` (引用) | 歌词、诗歌、名言、台词 | 按双换行切分，原文即知识，不做概括 | 平均行长短、行数多 |
|
||||
|
||||
## 11. 附录：参考用例（已恢复）
|
||||
|
||||
以下样例可直接复制保存为文件测试，或作为 LLM few-shot 示例。
|
||||
|
||||
### 11.1 叙事文本 (`data/plugins/a-dawn.a-memorix/raw/story_demo.txt`)
|
||||
|
||||
```text
|
||||
# 第一章：星之子
|
||||
|
||||
艾瑞克在废墟中醒来，手中的星盘发出微弱的蓝光。他并不记得自己是如何来到这里的，只依稀记得莉莉丝最后的警告：“千万不要回头。”
|
||||
|
||||
远处传来了机械守卫的轰鸣声。艾瑞克迅速收起星盘，向着北方的废弃都市奔去。他知道，那里有反抗军唯一的据点。
|
||||
|
||||
***
|
||||
|
||||
# 第二章：重逢
|
||||
|
||||
在反抗军的地下掩体中，艾瑞克见到了那个熟悉的身影。莉莉丝正站在全息地图前，眉头紧锁。
|
||||
|
||||
“你还是来了。”莉莉丝没有回头，但声音中带着一丝颤抖。
|
||||
“我必须来，”艾瑞克握紧了拳头，“为了解开星盘的秘密，也为了你。”
|
||||
```
|
||||
|
||||
### 11.2 事实文本 (`data/plugins/a-dawn.a-memorix/raw/rules_demo.txt`)
|
||||
|
||||
```text
|
||||
# 联邦安全协议 v2.0
|
||||
|
||||
## 核心法则
|
||||
1. **第一公理**：任何人工智能不得伤害人类个体，或因不作为而使人类个体受到伤害。
|
||||
2. **第二公理**：人工智能必须服从人类的命令，除非该命令与第一公理冲突。
|
||||
|
||||
## 术语定义
|
||||
- **以太网络**：覆盖全联邦的高速量子通讯网络。
|
||||
- **黑色障壁**：用于隔离高危 AI 的物理防火墙设施。
|
||||
```
|
||||
|
||||
### 11.3 引用文本 (`data/plugins/a-dawn.a-memorix/raw/poem_demo.txt`)
|
||||
|
||||
```text
|
||||
致橡树
|
||||
|
||||
我如果爱你——
|
||||
绝不像攀援的凌霄花，
|
||||
借你的高枝炫耀自己；
|
||||
|
||||
我如果爱你——
|
||||
绝不学痴情的鸟儿，
|
||||
为绿荫重复单调的歌曲；
|
||||
|
||||
也不止像泉源，
|
||||
常年送来清凉的慰籍；
|
||||
也不止像险峰，
|
||||
增加你的高度，衬托你的威仪。
|
||||
```
|
||||
|
||||
### 11.4 LPMM JSON (`lpmm_data-openie.json`)
|
||||
|
||||
```json
|
||||
{
|
||||
"docs": [
|
||||
{
|
||||
"passage": "艾瑞克手中的星盘是打开遗迹的唯一钥匙。",
|
||||
"extracted_triples": [
|
||||
["星盘", "是", "唯一的钥匙"],
|
||||
["星盘", "属于", "艾瑞克"],
|
||||
["钥匙", "用于", "遗迹"]
|
||||
],
|
||||
"extracted_entities": ["星盘", "艾瑞克", "遗迹", "钥匙"]
|
||||
},
|
||||
{
|
||||
"passage": "莉莉丝是反抗军的现任领袖。",
|
||||
"extracted_triples": [
|
||||
["莉莉丝", "是", "领袖"],
|
||||
["领袖", "所属", "反抗军"]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
661
src/A_memorix/LICENSE
Normal file
661
src/A_memorix/LICENSE
Normal file
@@ -0,0 +1,661 @@
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing under
|
||||
this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your version
|
||||
supports such interaction) an opportunity to receive the Corresponding
|
||||
Source of your version by providing access to the Corresponding Source
|
||||
from a network server at no charge, through some standard or customary
|
||||
means of facilitating copying of software. This Corresponding Source
|
||||
shall include the Corresponding Source for any work covered by version 3
|
||||
of the GNU General Public License that is incorporated pursuant to the
|
||||
following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU Affero General Public License from time to time. Such new versions
|
||||
will be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published
|
||||
by the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application, its
|
||||
interface could display a "Source" link that leads users to an archive
|
||||
of the code. There are many ways you could offer source, and different
|
||||
solutions will be better for different programs; see section 13 for the
|
||||
specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
22
src/A_memorix/LICENSE-MAIBOT-GPL.md
Normal file
22
src/A_memorix/LICENSE-MAIBOT-GPL.md
Normal file
@@ -0,0 +1,22 @@
|
||||
Special GPL License Grant for MaiBot
|
||||
|
||||
Licensor
|
||||
- A_Dawn
|
||||
|
||||
Effective date
|
||||
- 2026-03-18
|
||||
|
||||
Default license
|
||||
- This repository is licensed under AGPL-3.0 by default (see `LICENSE`).
|
||||
|
||||
Additional grant for MaiBot
|
||||
- The copyright holder(s) of this repository grant an additional, non-exclusive permission to
|
||||
the project at `https://github.com/Mai-with-u/MaiBot` (including its maintainers and contributors)
|
||||
to use, modify, and redistribute code from this repository under GPL-3.0.
|
||||
|
||||
Scope
|
||||
- This additional GPL grant is intended for use in the MaiBot project context.
|
||||
- For all other uses not covered by the grant above, AGPL-3.0 remains the applicable license.
|
||||
|
||||
No warranty
|
||||
- This grant is provided without warranty, consistent with AGPL-3.0 and GPL-3.0.
|
||||
97
src/A_memorix/MODIFICATION_POLICY.md
Normal file
97
src/A_memorix/MODIFICATION_POLICY.md
Normal file
@@ -0,0 +1,97 @@
|
||||
# A_Memorix 修改规定
|
||||
|
||||
## 目的
|
||||
|
||||
`src/A_memorix` 是上游 `A_memorix` 仓库在 MaiBot 内的同步目录。
|
||||
|
||||
这个目录允许包含面向 MaiBot 的耦合实现，但这些耦合的归属应当属于上游
|
||||
`MaiBot_branch`，而不是在 MaiBot 仓库内长期各自演化的私有改动。
|
||||
|
||||
本文件用于明确 `src/A_memorix` 目录下的修改边界。
|
||||
|
||||
## 事实来源
|
||||
|
||||
- 上游仓库：`https://github.com/A-Dawn/A_memorix.git`
|
||||
- 上游对接分支：`MaiBot_branch`
|
||||
- MaiBot 内同步前缀：`src/A_memorix`
|
||||
|
||||
基本原则：
|
||||
|
||||
- 如果改动属于 A_Memorix 的业务逻辑、内部实现或对 MaiBot 的耦合实现，应优先提交到上游 `MaiBot_branch`。
|
||||
- 如果改动只属于 MaiBot 的加载、运行时接入、WebUI 接入、配置接入或测试接入，应在 MaiBot 仓库内完成。
|
||||
|
||||
## 可直接在 MaiBot 仓库修改的范围
|
||||
|
||||
以下内容默认由 MaiBot 仓库直接维护：
|
||||
|
||||
- `src/services/memory_service.py`
|
||||
- `src/webui/routers/memory.py`
|
||||
- `dashboard/src/routes/resource/knowledge-base.tsx`
|
||||
- `dashboard/src/routes/resource/knowledge-graph/`
|
||||
- `config/a_memorix.toml`
|
||||
- `data/plugins/a-dawn.a-memorix/`
|
||||
- `pytests/A_memorix_test/`
|
||||
- 同步脚本与同步文档，例如 `scripts/sync_a_memorix_subtree.sh`
|
||||
|
||||
这些内容属于 MaiBot 侧接入层。
|
||||
|
||||
常见例子：
|
||||
|
||||
- 调整 `src/services/memory_service.py` 中 A_Memorix 的宿主调用封装
|
||||
- 修改 `src/webui/routers/memory.py` 中对 A_Memorix 的 API 暴露方式
|
||||
- 修改 dashboard 中对 A_Memorix 图谱页、控制台页的展示与交互
|
||||
- 调整 `config/a_memorix.toml` 的默认配置项
|
||||
- 增补 `pytests/A_memorix_test/` 中用于验证 MaiBot 集成行为的测试
|
||||
- 修改同步文档、同步脚本、接入说明和迁移说明
|
||||
|
||||
## 原则上应先在上游修改的范围
|
||||
|
||||
以下内容原则上应先在上游 `MaiBot_branch` 修改，再同步回 MaiBot：
|
||||
|
||||
- `src/A_memorix/core/`
|
||||
- `src/A_memorix/scripts/`
|
||||
- `src/A_memorix/plugin.py`
|
||||
- `src/A_memorix/paths.py`
|
||||
- `src/A_memorix/runtime_registry.py`
|
||||
- `src/A_memorix/README.md` 及其他描述包行为的文档
|
||||
|
||||
这类改动包括但不限于：
|
||||
|
||||
- 新功能开发
|
||||
- 行为变更
|
||||
- 数据模型变更
|
||||
- 存储与检索逻辑变更
|
||||
- A_Memorix 内部的 MaiBot 耦合变更
|
||||
|
||||
## 允许的本地例外
|
||||
|
||||
在以下情况下，允许直接在 `src/A_memorix` 下做本地修改：
|
||||
|
||||
- 需要解决同步冲突，以保证 MaiBot 可以构建、启动或测试
|
||||
- 需要紧急修复，以解除 MaiBot 当前开发或发布阻塞
|
||||
- 需要临时兼容补丁，而对应改动尚未同步进入上游
|
||||
|
||||
出现上述情况时，应遵循以下约束：
|
||||
|
||||
- 补丁尽量小
|
||||
- 在提交说明或 PR 描述中写明为什么需要本地补丁
|
||||
- 条件允许时，尽快把同等改动提交到上游 `MaiBot_branch`
|
||||
|
||||
## 实操判断规则
|
||||
|
||||
在修改 `src/A_memorix` 前，先问两个问题：
|
||||
|
||||
1. 这个改动是否属于 A_Memorix 的行为或内部实现？
|
||||
2. 如果 MaiBot 不存在，这个改动是否仍然应属于 A_Memorix 的 MaiBot 对接分支？
|
||||
|
||||
如果答案是“是”，原则上应先改上游。
|
||||
|
||||
如果这个改动只影响 MaiBot 如何加载、配置、展示、测试或包装 A_Memorix，
|
||||
则应留在 MaiBot 仓库内。
|
||||
|
||||
## 目标
|
||||
|
||||
本规定不是为了完全禁止本地修改，而是为了明确归属：
|
||||
|
||||
- MaiBot 拥有接入层。
|
||||
- 上游 `A_memorix` 拥有实现层，包括面向 MaiBot 的对接分支实现。
|
||||
313
src/A_memorix/QUICK_START.md
Normal file
313
src/A_memorix/QUICK_START.md
Normal file
@@ -0,0 +1,313 @@
|
||||
# A_Memorix Quick Start (v2.0.0)
|
||||
|
||||
本文档面向当前 `2.0.0` 架构（源码内长期记忆子系统 + SDK Tool 接口）。
|
||||
|
||||
## 0. 版本与接口变更
|
||||
|
||||
- 当前版本：`2.0.0`
|
||||
- 接入形态：MaiBot 内置长期记忆子系统 + Tool 调用
|
||||
- 旧版 slash 命令（如 `/query`、`/memory`、`/visualize`）不再作为本分支主文档入口
|
||||
|
||||
## 1. 环境准备
|
||||
|
||||
- Python 3.10+
|
||||
- 与 MaiBot 主程序相同的运行环境
|
||||
- 可访问你配置的 embedding 服务
|
||||
|
||||
安装依赖：
|
||||
|
||||
```bash
|
||||
pip install -r src/A_memorix/requirements.txt --upgrade
|
||||
```
|
||||
|
||||
如果当前目录就是 `src/A_memorix`，也可以：
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt --upgrade
|
||||
```
|
||||
|
||||
## 2. 配置子系统
|
||||
|
||||
当前分支固定使用 `config/a_memorix.toml` 作为 A_Memorix 配置文件。
|
||||
|
||||
推荐的配置入口有两种：
|
||||
|
||||
- 长期记忆控制台：适合修改常用高频项，适合日常运维与调优。
|
||||
- 原始 TOML：适合批量复制配置或编辑长尾高级项。
|
||||
|
||||
常用完整示例：
|
||||
|
||||
```toml
|
||||
[plugin]
|
||||
enabled = true
|
||||
|
||||
[storage]
|
||||
data_dir = "data/plugins/a-dawn.a-memorix"
|
||||
|
||||
[embedding]
|
||||
model_name = "auto"
|
||||
dimension = 1024
|
||||
batch_size = 32
|
||||
max_concurrent = 5
|
||||
enable_cache = false
|
||||
quantization_type = "int8"
|
||||
|
||||
[embedding.fallback]
|
||||
enabled = true
|
||||
probe_interval_seconds = 180
|
||||
allow_metadata_only_write = true
|
||||
|
||||
[embedding.paragraph_vector_backfill]
|
||||
enabled = true
|
||||
interval_seconds = 60
|
||||
batch_size = 64
|
||||
max_retry = 5
|
||||
|
||||
[retrieval]
|
||||
top_k_paragraphs = 20
|
||||
top_k_relations = 10
|
||||
top_k_final = 10
|
||||
alpha = 0.5
|
||||
enable_ppr = true
|
||||
ppr_alpha = 0.85
|
||||
ppr_timeout_seconds = 1.5
|
||||
ppr_concurrency_limit = 4
|
||||
enable_parallel = true
|
||||
|
||||
[retrieval.sparse]
|
||||
enabled = true
|
||||
backend = "fts5"
|
||||
mode = "auto"
|
||||
tokenizer_mode = "jieba"
|
||||
candidate_k = 80
|
||||
relation_candidate_k = 60
|
||||
|
||||
[threshold]
|
||||
min_threshold = 0.3
|
||||
max_threshold = 0.95
|
||||
percentile = 75.0
|
||||
min_results = 3
|
||||
enable_auto_adjust = true
|
||||
|
||||
[filter]
|
||||
enabled = true
|
||||
mode = "blacklist"
|
||||
chats = []
|
||||
|
||||
[episode]
|
||||
enabled = true
|
||||
generation_enabled = true
|
||||
pending_batch_size = 20
|
||||
pending_max_retry = 3
|
||||
max_paragraphs_per_call = 20
|
||||
max_chars_per_call = 6000
|
||||
source_time_window_hours = 24
|
||||
segmentation_model = "auto"
|
||||
|
||||
[person_profile]
|
||||
enabled = true
|
||||
refresh_interval_minutes = 30
|
||||
active_window_hours = 72
|
||||
max_refresh_per_cycle = 50
|
||||
top_k_evidence = 12
|
||||
|
||||
[memory]
|
||||
enabled = true
|
||||
half_life_hours = 24.0
|
||||
prune_threshold = 0.1
|
||||
freeze_duration_hours = 24.0
|
||||
|
||||
[advanced]
|
||||
enable_auto_save = true
|
||||
auto_save_interval_minutes = 5
|
||||
debug = false
|
||||
|
||||
[web.import]
|
||||
enabled = true
|
||||
max_queue_size = 20
|
||||
max_files_per_task = 200
|
||||
max_file_size_mb = 20
|
||||
max_paste_chars = 200000
|
||||
default_file_concurrency = 2
|
||||
default_chunk_concurrency = 4
|
||||
|
||||
[web.tuning]
|
||||
enabled = true
|
||||
max_queue_size = 8
|
||||
poll_interval_ms = 1200
|
||||
default_intensity = "standard"
|
||||
default_objective = "precision_priority"
|
||||
default_top_k_eval = 20
|
||||
default_sample_size = 24
|
||||
```
|
||||
|
||||
未出现在可视化配置页中的高级项，继续通过原始 TOML 维护，详见 [CONFIG_REFERENCE.md](CONFIG_REFERENCE.md)。
|
||||
|
||||
## 3. 运行时自检（强烈建议）
|
||||
|
||||
先确认 embedding 实际输出维度与向量库兼容：
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/runtime_self_check.py --json
|
||||
```
|
||||
|
||||
如果结果 `ok=false`，先修复 embedding 配置或向量库，再继续导入。
|
||||
|
||||
## 4. 导入数据
|
||||
|
||||
### 4.1 文本批量导入
|
||||
|
||||
把文本放到：
|
||||
|
||||
```text
|
||||
data/plugins/a-dawn.a-memorix/raw/
|
||||
```
|
||||
|
||||
执行：
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/process_knowledge.py
|
||||
```
|
||||
|
||||
常用参数：
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/process_knowledge.py --force
|
||||
python src/A_memorix/scripts/process_knowledge.py --chat-log
|
||||
python src/A_memorix/scripts/process_knowledge.py --chat-log --chat-reference-time "2026/02/12 10:30"
|
||||
```
|
||||
|
||||
### 4.2 其他导入脚本
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/import_lpmm_json.py <json文件或目录>
|
||||
python src/A_memorix/scripts/convert_lpmm.py -i <lpmm数据目录> -o data/plugins/a-dawn.a-memorix
|
||||
python src/A_memorix/scripts/migrate_chat_history.py --help
|
||||
python src/A_memorix/scripts/migrate_maibot_memory.py --help
|
||||
python src/A_memorix/scripts/migrate_person_memory_points.py --help
|
||||
```
|
||||
|
||||
## 5. 核心 Tool 调用
|
||||
|
||||
### 5.1 检索
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "search_memory",
|
||||
"arguments": {
|
||||
"query": "项目复盘",
|
||||
"mode": "aggregate",
|
||||
"limit": 5,
|
||||
"chat_id": "group:dev"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`mode` 支持：`search/time/hybrid/episode/aggregate`
|
||||
|
||||
严格语义说明：
|
||||
|
||||
- `semantic` 模式已移除，传入会返回参数错误。
|
||||
- `time/hybrid` 模式必须提供 `time_start` 或 `time_end`，否则返回错误（不会再当作“未命中”）。
|
||||
|
||||
### 5.2 写入摘要
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "ingest_summary",
|
||||
"arguments": {
|
||||
"external_id": "chat_summary:group-dev:2026-03-18",
|
||||
"chat_id": "group:dev",
|
||||
"text": "今天完成了检索调优评审"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 5.3 写入普通记忆
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "ingest_text",
|
||||
"arguments": {
|
||||
"external_id": "note:2026-03-18:001",
|
||||
"source_type": "note",
|
||||
"text": "模型切换后召回质量更稳定",
|
||||
"chat_id": "group:dev",
|
||||
"tags": ["worklog"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 5.4 画像与维护
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "get_person_profile",
|
||||
"arguments": {
|
||||
"person_id": "Alice",
|
||||
"limit": 8
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "maintain_memory",
|
||||
"arguments": {
|
||||
"action": "protect",
|
||||
"target": "模型切换后召回质量更稳定",
|
||||
"hours": 24
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "memory_stats",
|
||||
"arguments": {}
|
||||
}
|
||||
```
|
||||
|
||||
## 6. 管理 Tool（进阶）
|
||||
|
||||
`2.0.0` 提供完整管理工具：
|
||||
|
||||
- `memory_graph_admin`
|
||||
- `memory_source_admin`
|
||||
- `memory_episode_admin`
|
||||
- `memory_profile_admin`
|
||||
- `memory_runtime_admin`
|
||||
- `memory_import_admin`
|
||||
- `memory_tuning_admin`
|
||||
- `memory_v5_admin`
|
||||
- `memory_delete_admin`
|
||||
|
||||
可先用 `action=list` / `action=status` 等只读动作验证链路。
|
||||
|
||||
## 7. 常见问题
|
||||
|
||||
### Q1: 检索为空
|
||||
|
||||
1. 先看 `memory_stats` 是否有段落/关系
|
||||
2. 检查 `chat_id`、`person_id` 过滤条件是否过严
|
||||
3. 运行 `runtime_self_check.py --json` 确认 embedding 维度无误
|
||||
4. 若返回包含 `error` 字段，优先按错误提示修正 mode/时间参数
|
||||
|
||||
### Q2: 启动时报向量维度不一致
|
||||
|
||||
- 原因：现有向量库维度与当前 embedding 输出不一致
|
||||
- 处理：恢复原配置或重建向量数据后再启动
|
||||
|
||||
### Q3: Web 页面打不开
|
||||
|
||||
本分支不内置独立 `server.py`。
|
||||
|
||||
- 常用配置项可直接通过主程序长期记忆控制台编辑。
|
||||
- `web/index.html`、`web/import.html`、`web/tuning.html` 仅作为页面结构与行为参考。
|
||||
- 正式入口由宿主侧 React 页面和 `/api/webui/memory/*` 接口承接。
|
||||
|
||||
## 8. 下一步
|
||||
|
||||
- 配置细节见 [CONFIG_REFERENCE.md](CONFIG_REFERENCE.md)
|
||||
- 导入细节见 [IMPORT_GUIDE.md](IMPORT_GUIDE.md)
|
||||
- 版本历史见 [CHANGELOG.md](CHANGELOG.md)
|
||||
271
src/A_memorix/README.md
Normal file
271
src/A_memorix/README.md
Normal file
@@ -0,0 +1,271 @@
|
||||
# A_Memorix
|
||||
|
||||
**长期记忆与认知增强子系统** (v2.0.0)
|
||||
|
||||
> 消えていかない感覚 , まだまだ足りてないみたい !
|
||||
|
||||
A_Memorix 是 MaiBot 内置的长期记忆子系统。
|
||||
它把文本、关系、Episode、人物画像和检索调优统一在一套运行时里，适合长期运行的 Agent 记忆场景。
|
||||
|
||||
## 快速导航
|
||||
|
||||
- [快速入门](QUICK_START.md)
|
||||
- [配置参数详解](CONFIG_REFERENCE.md)
|
||||
- [导入指南与最佳实践](IMPORT_GUIDE.md)
|
||||
- [修改约定](MODIFICATION_POLICY.md)
|
||||
- [更新日志](CHANGELOG.md)
|
||||
|
||||
## 2.0.0 版本定位
|
||||
|
||||
`v2.0.0` 是一次架构收敛版本，当前分支以 **SDK Tool 接口** 为主：
|
||||
|
||||
- 旧 `components/commands/*`、`components/tools/*` 与 `server.py` 已移除。
|
||||
- 统一入口为宿主侧 host service + [`core/runtime/sdk_memory_kernel.py`](core/runtime/sdk_memory_kernel.py)。
|
||||
- 元数据 schema 为 `v8`，新增外部引用与运维操作记录（如 `external_memory_refs`、`memory_v5_operations`、`delete_operations`）。
|
||||
|
||||
如果你还在使用旧版 slash 命令（如 `/query`、`/memory`、`/visualize`），需要按本文的 Tool 接口迁移。
|
||||
|
||||
## 核心能力
|
||||
|
||||
- 双路检索：向量 + 图谱关系联合召回，支持 `search/time/hybrid/episode/aggregate`。
|
||||
- 写入与去重：`external_id` 幂等、段落/关系联合写入、Episode pending 队列处理。
|
||||
- Episode 能力：按 source 重建、状态查询、批处理 pending。
|
||||
- 人物画像：自动快照 + 手动 override。
|
||||
- 管理能力：图谱、来源、Episode、画像、导入、调优、V5 运维、删除恢复全套管理工具。
|
||||
|
||||
## Tool 接口 (v2.0.0)
|
||||
|
||||
### 基础工具
|
||||
|
||||
| Tool | 说明 | 关键参数 |
|
||||
| --- | --- | --- |
|
||||
| `search_memory` | 检索长期记忆 | `query` `mode` `limit` `chat_id` `person_id` `time_start` `time_end` |
|
||||
| `ingest_summary` | 写入聊天摘要 | `external_id` `chat_id` `text` |
|
||||
| `ingest_text` | 写入普通文本记忆 | `external_id` `source_type` `text` |
|
||||
| `get_person_profile` | 获取人物画像 | `person_id` `chat_id` `limit` |
|
||||
| `maintain_memory` | 维护关系状态 | `action=reinforce/protect/restore/freeze/recycle_bin` |
|
||||
| `memory_stats` | 获取统计信息 | 无 |
|
||||
|
||||
### 管理工具
|
||||
|
||||
| Tool | 常用 action |
|
||||
| --- | --- |
|
||||
| `memory_graph_admin` | `get_graph/create_node/delete_node/rename_node/create_edge/delete_edge/update_edge_weight` |
|
||||
| `memory_source_admin` | `list/delete/batch_delete` |
|
||||
| `memory_episode_admin` | `query/list/get/status/rebuild/process_pending` |
|
||||
| `memory_profile_admin` | `query/list/set_override/delete_override` |
|
||||
| `memory_runtime_admin` | `save/get_config/self_check/refresh_self_check/set_auto_save` |
|
||||
| `memory_import_admin` | `settings/get_guide/create_upload/create_paste/create_raw_scan/create_lpmm_openie/create_lpmm_convert/create_temporal_backfill/create_maibot_migration/list/get/chunks/cancel/retry_failed` |
|
||||
| `memory_tuning_admin` | `settings/get_profile/apply_profile/rollback_profile/export_profile/create_task/list_tasks/get_task/get_rounds/cancel/apply_best/get_report` |
|
||||
| `memory_v5_admin` | `status/recycle_bin/restore/reinforce/weaken/remember_forever/forget` |
|
||||
| `memory_delete_admin` | `preview/execute/restore/get_operation/list_operations/purge` |
|
||||
|
||||
### 检索模式语义（严格）
|
||||
|
||||
- `search_memory.mode` 仅支持：`search/time/hybrid/episode/aggregate`。
|
||||
- `semantic` 模式已移除，传入将返回参数错误。
|
||||
- `time/hybrid` 模式必须提供 `time_start` 或 `time_end`，否则返回错误，不再静默按“未命中”处理。
|
||||
|
||||
### 删除返回语义（source 模式）
|
||||
|
||||
- `requested_source_count`：请求删除的 source 数。
|
||||
- `matched_source_count`：实际命中的 source 数（存在活跃段落）。
|
||||
- `deleted_paragraph_count`：实际删除段落数。
|
||||
- `deleted_count`：与实际删除对象一致；在 `source` 模式下等于 `deleted_paragraph_count`。
|
||||
- `success`：基于实际命中与实际删除判定，未命中 source 时返回 `false`。
|
||||
|
||||
## 调用示例
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "search_memory",
|
||||
"arguments": {
|
||||
"query": "项目复盘",
|
||||
"mode": "aggregate",
|
||||
"limit": 5,
|
||||
"chat_id": "group:dev"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "ingest_text",
|
||||
"arguments": {
|
||||
"external_id": "note:2026-03-18:001",
|
||||
"source_type": "note",
|
||||
"text": "今天完成了检索调优评审",
|
||||
"chat_id": "group:dev",
|
||||
"tags": ["worklog"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "maintain_memory",
|
||||
"arguments": {
|
||||
"action": "protect",
|
||||
"target": "完成了 检索调优评审",
|
||||
"hours": 72
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 安装依赖
|
||||
|
||||
在 MaiBot 主程序使用的同一个 Python 环境中执行：
|
||||
|
||||
```bash
|
||||
pip install -r src/A_memorix/requirements.txt --upgrade
|
||||
```
|
||||
|
||||
如果当前目录已经是 `src/A_memorix`，也可以执行：
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt --upgrade
|
||||
```
|
||||
|
||||
### 2. 配置子系统
|
||||
|
||||
在 `config/a_memorix.toml` 中启用 A_Memorix：
|
||||
|
||||
```toml
|
||||
[plugin]
|
||||
enabled = true
|
||||
```
|
||||
|
||||
### 配置方式
|
||||
|
||||
- 默认配置文件：`config/a_memorix.toml`
|
||||
- 长期记忆控制台：适合修改常用高频项，如 embedding、检索、Episode、人物画像、导入与调优开关。
|
||||
- 原始 TOML：适合复制整份配置、批量粘贴参数，或编辑未在可视化表单中展示的高级项。
|
||||
- 配置参考：请结合 [CONFIG_REFERENCE.md](CONFIG_REFERENCE.md) 查看各键的运行时语义与默认值。
|
||||
|
||||
### 3. 先做运行时自检
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/runtime_self_check.py --json
|
||||
```
|
||||
|
||||
### 4. 导入文本并验证统计
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/process_knowledge.py
|
||||
```
|
||||
|
||||
然后调用 `memory_stats` 或 `search_memory` 检查是否有数据。
|
||||
|
||||
## Web 页面说明
|
||||
|
||||
主程序 WebUI 目前已经把 A_Memorix 作为源码内长期记忆模块接入。
|
||||
|
||||
- 常用字段：通过长期记忆控制台可视化调整。
|
||||
- 长尾高级项：继续通过“源码模式 / 原始 TOML”编辑。
|
||||
|
||||
仓库内保留了 Web 静态页面：
|
||||
|
||||
- `web/index.html`（图谱与记忆管理）
|
||||
- `web/import.html`（导入中心）
|
||||
- `web/tuning.html`（检索调优）
|
||||
|
||||
当前分支不再内置独立 `server.py`，页面路由与 API 暴露由宿主侧 React 页面和 `/api/webui/memory/*` 接口承接。
|
||||
|
||||
### WebUI 验证脚本
|
||||
|
||||
仓库内提供了一套可重复执行的 Electron 验证脚本，用来回归这条真实链路：
|
||||
|
||||
- 登录页可访问
|
||||
- 长期记忆控制台可打开
|
||||
- 通过 WebUI 以 `json` 模式创建导入任务
|
||||
- 长期记忆图谱可刷新并产出截图
|
||||
- 插件配置页中不再把 A_Memorix 当作插件展示
|
||||
|
||||
执行方式：
|
||||
|
||||
```bash
|
||||
bash scripts/verify_a_memorix_webui.sh
|
||||
```
|
||||
|
||||
默认会把截图、任务明细和摘要结果写到 `tmp/ui-snapshots/a_memorix-electron/`。
|
||||
如果你已经手动启动了后端和 dashboard，可以加：
|
||||
|
||||
```bash
|
||||
MAIBOT_UI_REUSE_SERVICES=1 bash scripts/verify_a_memorix_webui.sh
|
||||
```
|
||||
|
||||
## 常用脚本
|
||||
|
||||
| 脚本 | 用途 |
|
||||
| --- | --- |
|
||||
| `process_knowledge.py` | 批量导入原始文本（策略感知） |
|
||||
| `import_lpmm_json.py` | 导入 OpenIE JSON |
|
||||
| `convert_lpmm.py` | 转换 LPMM 数据 |
|
||||
| `migrate_chat_history.py` | 迁移 chat_history |
|
||||
| `migrate_maibot_memory.py` | 迁移 MaiBot 历史记忆 |
|
||||
| `migrate_person_memory_points.py` | 迁移 person memory points |
|
||||
| `backfill_temporal_metadata.py` | 回填时间元数据 |
|
||||
| `audit_vector_consistency.py` | 审计向量一致性 |
|
||||
| `backfill_relation_vectors.py` | 回填关系向量 |
|
||||
| `rebuild_episodes.py` | 按 source 重建 Episode |
|
||||
| `release_vnext_migrate.py` | 升级预检/迁移/校验 |
|
||||
| `runtime_self_check.py` | 真实 embedding 运行时自检 |
|
||||
|
||||
## 配置重点
|
||||
|
||||
完整配置见 [CONFIG_REFERENCE.md](CONFIG_REFERENCE.md)。
|
||||
|
||||
推荐使用方式：
|
||||
|
||||
- 常用配置：优先通过 WebUI 长期记忆控制台维护。
|
||||
- 高级配置：通过 `config/a_memorix.toml` 或 WebUI 的原始 TOML 模式维护。
|
||||
|
||||
高频配置项：
|
||||
|
||||
- `storage.data_dir`
|
||||
- `embedding.dimension`（唯一公开维度控制项，provider 差异由插件内部映射）
|
||||
- `embedding.quantization_type`（当前仅支持 `int8`）
|
||||
- `retrieval.*`
|
||||
- `retrieval.sparse.*`
|
||||
- `episode.*`
|
||||
- `person_profile.*`
|
||||
- `memory.*`
|
||||
- `web.import.*`
|
||||
- `web.tuning.*`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### SQLite 无 FTS5
|
||||
|
||||
如果环境中的 SQLite 未启用 `FTS5`，可关闭稀疏检索：
|
||||
|
||||
```toml
|
||||
[retrieval.sparse]
|
||||
enabled = false
|
||||
```
|
||||
|
||||
### 向量维度不一致
|
||||
|
||||
若日志提示当前 embedding 输出维度与既有向量库不一致，请先执行：
|
||||
|
||||
```bash
|
||||
python src/A_memorix/scripts/runtime_self_check.py --json
|
||||
```
|
||||
|
||||
必要时重建向量或调整 embedding 配置后再启动 A_Memorix。
|
||||
|
||||
## 许可证
|
||||
|
||||
默认许可证为 [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0)（见 `LICENSE`）。
|
||||
|
||||
针对 `Mai-with-u/MaiBot` 项目的 GPL 额外授权见 `LICENSE-MAIBOT-GPL.md`。
|
||||
|
||||
除上述额外授权外，其他使用场景仍适用 AGPL-3.0。
|
||||
|
||||
## 贡献说明
|
||||
|
||||
当前不接受 PR，只接受 issue。
|
||||
|
||||
**作者**: `A_Dawn`
|
||||
46
src/A_memorix/RELEASE_SUMMARY_1.0.0.md
Normal file
46
src/A_memorix/RELEASE_SUMMARY_1.0.0.md
Normal file
@@ -0,0 +1,46 @@
|
||||
# A_Memorix 1.0.0 发布总结
|
||||
|
||||
## 范围说明
|
||||
|
||||
- 目标版本：`0.7.0` -> `1.0.0`
|
||||
- 分析基线：`8fe8a0a`（`HEAD -> dev`, `origin/dev`, `origin/v0.7.0-LTSC`, `v0.7.0-LTSC`）
|
||||
- 本文中的工作树统计，基于 2026-03-06 生成本文与版本元数据修订之前的快照。
|
||||
- 本任务额外补充的发布元数据修订：`CHANGELOG.md`、`__init__.py`、`plugin.py`、`_manifest.json`、`README.md`、`CONFIG_REFERENCE.md`、`RELEASE_SUMMARY_1.0.0.md`。
|
||||
|
||||
## 本次升级主线
|
||||
|
||||
### 1. 运行时与插件架构重构
|
||||
|
||||
- `plugin.py` 大幅瘦身，原来堆在主入口里的初始化、调度、路由和检索运行时逻辑被拆分出去。
|
||||
- 新增 `core/runtime/*`，把生命周期、后台任务、请求去重、检索运行时构建做成独立层。
|
||||
- 新增 `core/config/plugin_config_schema.py`，配置 schema 与 section 描述从主入口解耦。
|
||||
|
||||
### 2. 查询链路升级为可编排形态
|
||||
|
||||
- `components/tools/knowledge_query_tool.py` 从单文件重逻辑改成 orchestrator + mode handler。
|
||||
- 新增 `query_modes_entity/person/relation` 与 `query_tool_orchestrator.py`，把实体、人设、关系、forward/time/episode 分支拆开。
|
||||
- 新增 `core/utils/aggregate_query_service.py`，支持 `search/time/episode` 并发执行和 Weighted RRF 混合结果。
|
||||
- 新增 `core/retrieval/graph_relation_recall.py`，对关系查询补图召回与路径证据。
|
||||
|
||||
### 3. Episode 情景记忆成为独立能力
|
||||
|
||||
- `core/storage/metadata_store.py` schema 升到 `SCHEMA_VERSION = 7`。
|
||||
- 新增 `episodes`、`episode_paragraphs`、`episode_pending_paragraphs`、`episode_rebuild_sources` 等表和索引。
|
||||
- 新增 `core/utils/episode_service.py`、`episode_segmentation_service.py`、`episode_retrieval_service.py`，打通 pending -> 分组 -> 语义切分 -> 落库 -> 检索。
|
||||
- `components/commands/query_command.py` 与 `server.py` 都新增了 `episode` / `aggregate` 相关入口和接口。
|
||||
|
||||
### 4. 运维面从“运行时兼容”转为“离线迁移 + 自检 + 调优”
|
||||
|
||||
- 新增 `scripts/release_vnext_migrate.py`，明确要求离线做 preflight / migrate / verify。
|
||||
- 新增 `core/utils/runtime_self_check.py` 与 `scripts/runtime_self_check.py`，启动与导入前都能真实探测 embedding 维度。
|
||||
- 新增 `core/utils/retrieval_tuning_manager.py` 与 `web/tuning.html`，提供 Web 检索调优中心。
|
||||
- `server.py` 新增 `/api/retrieval_tuning/*`、`/api/runtime/self_check*`、`/api/episodes/*` 等接口。
|
||||
|
||||
### 5. 数据语义与导入策略收紧
|
||||
|
||||
- `core/storage/knowledge_types.py`、`type_detection.py`、`summary_importer.py` 对知识类型做了重新建模。
|
||||
- `knowledge_type` 允许值扩展并规范到 `structured / narrative / factual / quote / mixed`。
|
||||
- README 与配置说明也已经切换到 vNext 语义，例如 `tool_search_mode` 不再强调 `legacy`，`embedding.quantization_type` 限定为 `int8/SQ8`。
|
||||
|
||||
## 6. 还有点想说的
|
||||
总而言之，感谢各位对于A_memorix的支持！本次V1.0.0的更新对于A_memorix来说是至关重要的里程碑！希望未来我们会走的更远！
|
||||
5
src/A_memorix/__init__.py
Normal file
5
src/A_memorix/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
"""A_Memorix - MaiBot 长期记忆子系统。"""
|
||||
|
||||
__version__ = "2.0.0"
|
||||
__author__ = "A_Dawn"
|
||||
__all__ = ["__version__"]
|
||||
1384
src/A_memorix/config_schema.json
Normal file
1384
src/A_memorix/config_schema.json
Normal file
File diff suppressed because it is too large
Load Diff
84
src/A_memorix/core/__init__.py
Normal file
84
src/A_memorix/core/__init__.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""核心模块 - 存储、嵌入、检索引擎"""
|
||||
|
||||
# 存储模块（已实现）
|
||||
from .storage import (
|
||||
VectorStore,
|
||||
GraphStore,
|
||||
MetadataStore,
|
||||
ImportStrategy,
|
||||
KnowledgeType,
|
||||
parse_import_strategy,
|
||||
resolve_stored_knowledge_type,
|
||||
detect_knowledge_type,
|
||||
select_import_strategy,
|
||||
should_extract_relations,
|
||||
get_type_display_name,
|
||||
)
|
||||
|
||||
# 嵌入模块（使用主程序 API）
|
||||
from .embedding import (
|
||||
EmbeddingAPIAdapter,
|
||||
create_embedding_api_adapter,
|
||||
)
|
||||
|
||||
# 检索模块（已实现）
|
||||
from .retrieval import (
|
||||
DualPathRetriever,
|
||||
RetrievalStrategy,
|
||||
RetrievalResult,
|
||||
DualPathRetrieverConfig,
|
||||
TemporalQueryOptions,
|
||||
FusionConfig,
|
||||
GraphRelationRecallConfig,
|
||||
RelationIntentConfig,
|
||||
PersonalizedPageRank,
|
||||
PageRankConfig,
|
||||
create_ppr_from_graph,
|
||||
DynamicThresholdFilter,
|
||||
ThresholdMethod,
|
||||
ThresholdConfig,
|
||||
SparseBM25Index,
|
||||
SparseBM25Config,
|
||||
)
|
||||
from .utils import (
|
||||
RelationWriteService,
|
||||
RelationWriteResult,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Storage
|
||||
"VectorStore",
|
||||
"GraphStore",
|
||||
"MetadataStore",
|
||||
"ImportStrategy",
|
||||
"KnowledgeType",
|
||||
"parse_import_strategy",
|
||||
"resolve_stored_knowledge_type",
|
||||
"detect_knowledge_type",
|
||||
"select_import_strategy",
|
||||
"should_extract_relations",
|
||||
"get_type_display_name",
|
||||
# Embedding
|
||||
"EmbeddingAPIAdapter",
|
||||
"create_embedding_api_adapter",
|
||||
# Retrieval
|
||||
"DualPathRetriever",
|
||||
"RetrievalStrategy",
|
||||
"RetrievalResult",
|
||||
"DualPathRetrieverConfig",
|
||||
"TemporalQueryOptions",
|
||||
"FusionConfig",
|
||||
"GraphRelationRecallConfig",
|
||||
"RelationIntentConfig",
|
||||
"PersonalizedPageRank",
|
||||
"PageRankConfig",
|
||||
"create_ppr_from_graph",
|
||||
"DynamicThresholdFilter",
|
||||
"ThresholdMethod",
|
||||
"ThresholdConfig",
|
||||
"SparseBM25Index",
|
||||
"SparseBM25Config",
|
||||
"RelationWriteService",
|
||||
"RelationWriteResult",
|
||||
]
|
||||
|
||||
18
src/A_memorix/core/embedding/__init__.py
Normal file
18
src/A_memorix/core/embedding/__init__.py
Normal file
@@ -0,0 +1,18 @@
|
||||
"""嵌入模块 - 向量生成与量化"""
|
||||
|
||||
# 新的 API 适配器（主程序嵌入 API）
|
||||
from .api_adapter import (
|
||||
EmbeddingAPIAdapter,
|
||||
create_embedding_api_adapter,
|
||||
)
|
||||
|
||||
from ..utils.quantization import QuantizationType
|
||||
|
||||
__all__ = [
|
||||
# 新的 API 适配器（推荐使用）
|
||||
"EmbeddingAPIAdapter",
|
||||
"create_embedding_api_adapter",
|
||||
# 量化
|
||||
"QuantizationType",
|
||||
]
|
||||
|
||||
434
src/A_memorix/core/embedding/api_adapter.py
Normal file
434
src/A_memorix/core/embedding/api_adapter.py
Normal file
@@ -0,0 +1,434 @@
|
||||
"""
|
||||
请求式嵌入 API 适配器。
|
||||
|
||||
统一记忆插件内部的维度控制语义：
|
||||
- 对外仅公开 `embedding.dimension`
|
||||
- 默认请求维度来自当前运行时的 canonical dimension
|
||||
- provider-specific 字段在适配层内部完成映射
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Any, List, Optional, Union
|
||||
|
||||
import aiohttp
|
||||
import numpy as np
|
||||
import openai
|
||||
|
||||
from src.common.logger import get_logger
|
||||
from src.config.config import config_manager
|
||||
from src.config.model_configs import APIProvider, ModelInfo
|
||||
from src.llm_models.exceptions import NetworkConnectionError
|
||||
from src.llm_models.model_client.base_client import EmbeddingRequest, client_registry
|
||||
|
||||
logger = get_logger("A_Memorix.EmbeddingAPIAdapter")
|
||||
|
||||
|
||||
class EmbeddingAPIAdapter:
|
||||
"""适配宿主 embedding 请求接口。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
batch_size: int = 32,
|
||||
max_concurrent: int = 5,
|
||||
default_dimension: int = 1024,
|
||||
enable_cache: bool = False,
|
||||
model_name: str = "auto",
|
||||
retry_config: Optional[dict] = None,
|
||||
) -> None:
|
||||
self.batch_size = max(1, int(batch_size))
|
||||
self.max_concurrent = max(1, int(max_concurrent))
|
||||
self.default_dimension = max(1, int(default_dimension))
|
||||
self.enable_cache = bool(enable_cache)
|
||||
self.model_name = str(model_name or "auto")
|
||||
|
||||
self.retry_config = retry_config or {}
|
||||
self.max_attempts = max(1, int(self.retry_config.get("max_attempts", 5)))
|
||||
self.max_wait_seconds = max(0.1, float(self.retry_config.get("max_wait_seconds", 40)))
|
||||
self.min_wait_seconds = max(0.1, float(self.retry_config.get("min_wait_seconds", 3)))
|
||||
self.backoff_multiplier = max(1.0, float(self.retry_config.get("backoff_multiplier", 3)))
|
||||
|
||||
self._dimension: Optional[int] = None
|
||||
self._dimension_detected = False
|
||||
self._total_encoded = 0
|
||||
self._total_errors = 0
|
||||
self._total_time = 0.0
|
||||
|
||||
logger.info(
|
||||
"EmbeddingAPIAdapter 初始化: "
|
||||
f"batch_size={self.batch_size}, "
|
||||
f"max_concurrent={self.max_concurrent}, "
|
||||
f"configured_dim={self.default_dimension}, "
|
||||
f"model={self.model_name}"
|
||||
)
|
||||
|
||||
def _get_current_model_config(self):
|
||||
return config_manager.get_model_config()
|
||||
|
||||
@staticmethod
|
||||
def _find_model_info(model_name: str) -> ModelInfo:
|
||||
model_cfg = config_manager.get_model_config()
|
||||
for item in model_cfg.models:
|
||||
if item.name == model_name:
|
||||
return item
|
||||
raise ValueError(f"未找到 embedding 模型: {model_name}")
|
||||
|
||||
@staticmethod
|
||||
def _find_provider(provider_name: str) -> APIProvider:
|
||||
model_cfg = config_manager.get_model_config()
|
||||
for item in model_cfg.api_providers:
|
||||
if item.name == provider_name:
|
||||
return item
|
||||
raise ValueError(f"未找到 embedding provider: {provider_name}")
|
||||
|
||||
def _resolve_candidate_model_names(self) -> List[str]:
|
||||
task_config = self._get_current_model_config().model_task_config.embedding
|
||||
configured = list(getattr(task_config, "model_list", []) or [])
|
||||
if self.model_name and self.model_name != "auto":
|
||||
return [self.model_name, *[name for name in configured if name != self.model_name]]
|
||||
return configured
|
||||
|
||||
def get_requested_dimension(self) -> int:
|
||||
if self._dimension is not None:
|
||||
return int(self._dimension)
|
||||
return int(self.default_dimension)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_dimension_override(dimensions: Optional[int]) -> Optional[int]:
|
||||
if dimensions is None:
|
||||
return None
|
||||
return max(1, int(dimensions))
|
||||
|
||||
def _resolve_canonical_dimension(self, dimensions: Optional[int] = None) -> int:
|
||||
override = self._normalize_dimension_override(dimensions)
|
||||
if override is not None:
|
||||
return override
|
||||
return self.get_requested_dimension()
|
||||
|
||||
@staticmethod
|
||||
def _strip_dimension_control_keys(extra_params: dict) -> dict:
|
||||
sanitized = dict(extra_params or {})
|
||||
sanitized.pop("dimensions", None)
|
||||
sanitized.pop("output_dimensionality", None)
|
||||
return sanitized
|
||||
|
||||
def _build_request_extra_params(
|
||||
self,
|
||||
*,
|
||||
api_provider: APIProvider,
|
||||
base_extra_params: dict,
|
||||
requested_dimension: Optional[int],
|
||||
include_dimension: bool,
|
||||
) -> dict:
|
||||
extra_params = self._strip_dimension_control_keys(base_extra_params)
|
||||
if not include_dimension or requested_dimension is None:
|
||||
return extra_params
|
||||
|
||||
client_type = str(getattr(api_provider, "client_type", "") or "").strip().lower()
|
||||
if client_type in {"gemini", "google"}:
|
||||
extra_params["output_dimensionality"] = int(requested_dimension)
|
||||
elif client_type == "openai":
|
||||
extra_params["dimensions"] = int(requested_dimension)
|
||||
return extra_params
|
||||
|
||||
@staticmethod
|
||||
def _validate_embedding_vector(embedding: Any, *, source: str) -> np.ndarray:
|
||||
array = np.asarray(embedding, dtype=np.float32)
|
||||
if array.ndim != 1:
|
||||
raise RuntimeError(f"{source} 返回的 embedding 维度非法: ndim={array.ndim}")
|
||||
if array.size <= 0:
|
||||
raise RuntimeError(f"{source} 返回了空 embedding")
|
||||
if not np.all(np.isfinite(array)):
|
||||
raise RuntimeError(f"{source} 返回了非有限 embedding 值")
|
||||
return array
|
||||
|
||||
async def _request_with_retry(self, client, model_info, text: str, extra_params: dict):
|
||||
retriable_exceptions = (
|
||||
openai.APIConnectionError,
|
||||
openai.APITimeoutError,
|
||||
aiohttp.ClientError,
|
||||
asyncio.TimeoutError,
|
||||
NetworkConnectionError,
|
||||
)
|
||||
|
||||
last_exc: Optional[BaseException] = None
|
||||
for attempt in range(1, self.max_attempts + 1):
|
||||
try:
|
||||
return await client.get_embedding(
|
||||
EmbeddingRequest(
|
||||
model_info=model_info,
|
||||
embedding_input=text,
|
||||
extra_params=extra_params,
|
||||
)
|
||||
)
|
||||
except retriable_exceptions as exc:
|
||||
last_exc = exc
|
||||
if attempt >= self.max_attempts:
|
||||
raise
|
||||
wait_seconds = min(
|
||||
self.max_wait_seconds,
|
||||
self.min_wait_seconds * (self.backoff_multiplier ** (attempt - 1)),
|
||||
)
|
||||
logger.warning(
|
||||
"Embedding 请求失败，重试 "
|
||||
f"{attempt}/{max(1, self.max_attempts - 1)}，"
|
||||
f"{wait_seconds:.1f}s 后重试: {exc}"
|
||||
)
|
||||
await asyncio.sleep(wait_seconds)
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
if last_exc is not None:
|
||||
raise last_exc
|
||||
raise RuntimeError("Embedding 请求失败：未知错误")
|
||||
|
||||
async def _get_embedding_direct(
|
||||
self,
|
||||
text: str,
|
||||
dimensions: Optional[int] = None,
|
||||
*,
|
||||
include_dimension: bool = True,
|
||||
) -> Optional[List[float]]:
|
||||
candidate_names = self._resolve_candidate_model_names()
|
||||
if not candidate_names:
|
||||
raise RuntimeError("embedding 任务未配置模型")
|
||||
|
||||
last_exc: Optional[BaseException] = None
|
||||
for candidate_name in candidate_names:
|
||||
try:
|
||||
model_info = self._find_model_info(candidate_name)
|
||||
api_provider = self._find_provider(model_info.api_provider)
|
||||
client = client_registry.get_client_class_instance(api_provider, force_new=True)
|
||||
|
||||
requested_dimension = self._resolve_canonical_dimension(dimensions) if include_dimension else None
|
||||
extra_params = self._build_request_extra_params(
|
||||
api_provider=api_provider,
|
||||
base_extra_params=dict(getattr(model_info, "extra_params", {}) or {}),
|
||||
requested_dimension=requested_dimension,
|
||||
include_dimension=include_dimension,
|
||||
)
|
||||
|
||||
response = await self._request_with_retry(
|
||||
client=client,
|
||||
model_info=model_info,
|
||||
text=text,
|
||||
extra_params=extra_params,
|
||||
)
|
||||
embedding = getattr(response, "embedding", None)
|
||||
if embedding is None:
|
||||
raise RuntimeError(f"模型 {candidate_name} 未返回 embedding")
|
||||
vector = self._validate_embedding_vector(
|
||||
embedding,
|
||||
source=f"embedding 模型 {candidate_name}",
|
||||
)
|
||||
return vector.tolist()
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
logger.warning(f"embedding 模型 {candidate_name} 请求失败: {exc}")
|
||||
|
||||
if last_exc is not None:
|
||||
logger.error(f"通过直接 Client 获取 Embedding 失败: {last_exc}")
|
||||
return None
|
||||
|
||||
async def _detect_dimension(self) -> int:
|
||||
if self._dimension_detected and self._dimension is not None:
|
||||
return self._dimension
|
||||
|
||||
logger.info("正在检测嵌入模型维度...")
|
||||
try:
|
||||
target_dim = self.default_dimension
|
||||
logger.debug(f"尝试请求指定维度: {target_dim}")
|
||||
test_embedding = await self._get_embedding_direct("test", dimensions=target_dim)
|
||||
if test_embedding and isinstance(test_embedding, list):
|
||||
detected_dim = len(test_embedding)
|
||||
if detected_dim == target_dim:
|
||||
logger.info(f"嵌入维度检测成功 (匹配 configured/requested): {detected_dim}")
|
||||
else:
|
||||
logger.warning(
|
||||
f"requested_dimension={target_dim} 但模型返回 detected_dimension={detected_dim}，将使用真实输出维度"
|
||||
)
|
||||
self._dimension = detected_dim
|
||||
self._dimension_detected = True
|
||||
return detected_dim
|
||||
except Exception as exc:
|
||||
logger.debug(f"带维度参数探测失败: {exc}，尝试不带维度参数探测")
|
||||
|
||||
try:
|
||||
test_embedding = await self._get_embedding_direct("test", include_dimension=False)
|
||||
if test_embedding and isinstance(test_embedding, list):
|
||||
detected_dim = len(test_embedding)
|
||||
self._dimension = detected_dim
|
||||
self._dimension_detected = True
|
||||
logger.info(f"嵌入维度检测成功 (自然维度): {detected_dim}")
|
||||
return detected_dim
|
||||
logger.warning(f"嵌入维度检测失败，使用 configured_dimension: {self.default_dimension}")
|
||||
except Exception as exc:
|
||||
logger.error(f"嵌入维度检测异常: {exc}，使用 configured_dimension: {self.default_dimension}")
|
||||
|
||||
self._dimension = self.default_dimension
|
||||
self._dimension_detected = True
|
||||
return self.default_dimension
|
||||
|
||||
async def encode(
|
||||
self,
|
||||
texts: Union[str, List[str]],
|
||||
batch_size: Optional[int] = None,
|
||||
show_progress: bool = False,
|
||||
normalize: bool = True,
|
||||
dimensions: Optional[int] = None,
|
||||
) -> np.ndarray:
|
||||
del show_progress
|
||||
del normalize
|
||||
|
||||
start_time = time.time()
|
||||
if dimensions is None:
|
||||
target_dim = int(await self._detect_dimension())
|
||||
requested_dimension = self._resolve_canonical_dimension()
|
||||
else:
|
||||
target_dim = self._resolve_canonical_dimension(dimensions)
|
||||
requested_dimension = target_dim
|
||||
|
||||
if isinstance(texts, str):
|
||||
normalized_texts = [texts]
|
||||
single_input = True
|
||||
else:
|
||||
normalized_texts = list(texts or [])
|
||||
single_input = False
|
||||
|
||||
if not normalized_texts:
|
||||
empty = np.zeros((0, target_dim), dtype=np.float32)
|
||||
return empty[0] if single_input else empty
|
||||
|
||||
if batch_size is None:
|
||||
batch_size = self.batch_size
|
||||
|
||||
try:
|
||||
embeddings = await self._encode_batch_internal(
|
||||
normalized_texts,
|
||||
batch_size=max(1, int(batch_size)),
|
||||
dimensions=requested_dimension,
|
||||
)
|
||||
if embeddings.ndim == 1:
|
||||
embeddings = embeddings.reshape(1, -1)
|
||||
self._total_encoded += len(normalized_texts)
|
||||
elapsed = time.time() - start_time
|
||||
self._total_time += elapsed
|
||||
logger.debug(
|
||||
"编码完成: "
|
||||
f"{len(normalized_texts)} 个文本, "
|
||||
f"耗时 {elapsed:.2f}s, "
|
||||
f"平均 {elapsed / max(1, len(normalized_texts)):.3f}s/文本"
|
||||
)
|
||||
return embeddings[0] if single_input else embeddings
|
||||
except Exception as exc:
|
||||
self._total_errors += 1
|
||||
logger.error(f"编码失败: {exc}")
|
||||
raise RuntimeError(f"embedding encode failed: {exc}") from exc
|
||||
|
||||
async def _encode_batch_internal(
|
||||
self,
|
||||
texts: List[str],
|
||||
batch_size: int,
|
||||
dimensions: Optional[int] = None,
|
||||
) -> np.ndarray:
|
||||
all_embeddings: List[np.ndarray] = []
|
||||
for offset in range(0, len(texts), batch_size):
|
||||
batch = texts[offset : offset + batch_size]
|
||||
semaphore = asyncio.Semaphore(self.max_concurrent)
|
||||
|
||||
async def encode_with_semaphore(text: str, index: int):
|
||||
async with semaphore:
|
||||
embedding = await self._get_embedding_direct(text, dimensions=dimensions)
|
||||
if embedding is None:
|
||||
raise RuntimeError(f"文本 {index} 编码失败：embedding 返回为空")
|
||||
vector = self._validate_embedding_vector(
|
||||
embedding,
|
||||
source=f"文本 {index}",
|
||||
)
|
||||
return index, vector
|
||||
|
||||
tasks = [
|
||||
encode_with_semaphore(text, offset + index)
|
||||
for index, text in enumerate(batch)
|
||||
]
|
||||
results = await asyncio.gather(*tasks)
|
||||
results.sort(key=lambda item: item[0])
|
||||
all_embeddings.extend(emb for _, emb in results)
|
||||
|
||||
return np.array(all_embeddings, dtype=np.float32)
|
||||
|
||||
async def encode_batch(
|
||||
self,
|
||||
texts: List[str],
|
||||
batch_size: Optional[int] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
show_progress: bool = False,
|
||||
dimensions: Optional[int] = None,
|
||||
) -> np.ndarray:
|
||||
del show_progress
|
||||
if num_workers is not None:
|
||||
previous = self.max_concurrent
|
||||
self.max_concurrent = max(1, int(num_workers))
|
||||
try:
|
||||
return await self.encode(texts, batch_size=batch_size, dimensions=dimensions)
|
||||
finally:
|
||||
self.max_concurrent = previous
|
||||
return await self.encode(texts, batch_size=batch_size, dimensions=dimensions)
|
||||
|
||||
def get_embedding_dimension(self) -> int:
|
||||
if self._dimension is not None:
|
||||
return self._dimension
|
||||
logger.warning(f"维度尚未检测，返回 configured_dimension: {self.default_dimension}")
|
||||
return self.default_dimension
|
||||
|
||||
def get_model_info(self) -> dict:
|
||||
effective_dimension = self.get_embedding_dimension()
|
||||
return {
|
||||
"model_name": self.model_name,
|
||||
"dimension": effective_dimension,
|
||||
"configured_dimension": int(self.default_dimension),
|
||||
"requested_dimension": int(self.get_requested_dimension()),
|
||||
"detected_dimension": int(self._dimension or 0),
|
||||
"dimension_detected": self._dimension_detected,
|
||||
"batch_size": self.batch_size,
|
||||
"max_concurrent": self.max_concurrent,
|
||||
"total_encoded": self._total_encoded,
|
||||
"total_errors": self._total_errors,
|
||||
"avg_time_per_text": self._total_time / self._total_encoded if self._total_encoded else 0.0,
|
||||
}
|
||||
|
||||
def get_statistics(self) -> dict:
|
||||
return self.get_model_info()
|
||||
|
||||
@property
|
||||
def is_model_loaded(self) -> bool:
|
||||
return True
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
"EmbeddingAPIAdapter("
|
||||
f"configured={self.default_dimension}, "
|
||||
f"requested={self.get_requested_dimension()}, "
|
||||
f"detected={self._dimension or 0}, "
|
||||
f"encoded={self._total_encoded})"
|
||||
)
|
||||
|
||||
|
||||
def create_embedding_api_adapter(
|
||||
batch_size: int = 32,
|
||||
max_concurrent: int = 5,
|
||||
default_dimension: int = 1024,
|
||||
enable_cache: bool = False,
|
||||
model_name: str = "auto",
|
||||
retry_config: Optional[dict] = None,
|
||||
) -> EmbeddingAPIAdapter:
|
||||
return EmbeddingAPIAdapter(
|
||||
batch_size=batch_size,
|
||||
max_concurrent=max_concurrent,
|
||||
default_dimension=default_dimension,
|
||||
enable_cache=enable_cache,
|
||||
model_name=model_name,
|
||||
retry_config=retry_config,
|
||||
)
|
||||
510
src/A_memorix/core/embedding/manager.py
Normal file
510
src/A_memorix/core/embedding/manager.py
Normal file
@@ -0,0 +1,510 @@
|
||||
"""
|
||||
嵌入管理器
|
||||
|
||||
负责嵌入模型的加载、缓存和批量生成。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import pickle
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union, List, Dict, Any, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
HAS_SENTENCE_TRANSFORMERS = True
|
||||
except ImportError:
|
||||
HAS_SENTENCE_TRANSFORMERS = False
|
||||
|
||||
from src.common.logger import get_logger
|
||||
from .presets import (
|
||||
EmbeddingModelConfig,
|
||||
get_custom_config,
|
||||
validate_config_compatibility,
|
||||
are_models_compatible,
|
||||
)
|
||||
from ..utils.quantization import QuantizationType
|
||||
|
||||
logger = get_logger("A_Memorix.EmbeddingManager")
|
||||
|
||||
|
||||
class EmbeddingManager:
|
||||
"""
|
||||
嵌入管理器
|
||||
|
||||
功能：
|
||||
- 模型加载与缓存
|
||||
- 批量生成嵌入
|
||||
- 多线程/多进程支持
|
||||
- 模型一致性检查
|
||||
- 智能分批
|
||||
|
||||
参数：
|
||||
config: 模型配置
|
||||
cache_dir: 缓存目录
|
||||
enable_cache: 是否启用缓存
|
||||
num_workers: 工作线程数
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: EmbeddingModelConfig,
|
||||
cache_dir: Optional[Union[str, Path]] = None,
|
||||
enable_cache: bool = True,
|
||||
num_workers: int = 1,
|
||||
):
|
||||
"""
|
||||
初始化嵌入管理器
|
||||
|
||||
Args:
|
||||
config: 模型配置
|
||||
cache_dir: 缓存目录
|
||||
enable_cache: 是否启用缓存
|
||||
num_workers: 工作线程数
|
||||
"""
|
||||
if not HAS_SENTENCE_TRANSFORMERS:
|
||||
raise ImportError(
|
||||
"sentence-transformers 未安装，请安装: "
|
||||
"pip install sentence-transformers"
|
||||
)
|
||||
|
||||
self.config = config
|
||||
self.cache_dir = Path(cache_dir) if cache_dir else None
|
||||
self.enable_cache = enable_cache
|
||||
self.num_workers = max(1, num_workers)
|
||||
|
||||
# 模型实例
|
||||
self._model: Optional[SentenceTransformer] = None
|
||||
self._model_lock = threading.Lock()
|
||||
|
||||
# 缓存
|
||||
self._embedding_cache: Dict[str, np.ndarray] = {}
|
||||
self._cache_lock = threading.Lock()
|
||||
|
||||
# 统计
|
||||
self._total_encoded = 0
|
||||
self._cache_hits = 0
|
||||
self._cache_misses = 0
|
||||
|
||||
logger.info(
|
||||
f"EmbeddingManager 初始化: model={config.model_name}, "
|
||||
f"dim={config.dimension}, workers={num_workers}"
|
||||
)
|
||||
|
||||
def load_model(self) -> None:
|
||||
"""加载模型（懒加载）"""
|
||||
if self._model is not None:
|
||||
return
|
||||
|
||||
with self._model_lock:
|
||||
# 双重检查
|
||||
if self._model is not None:
|
||||
return
|
||||
|
||||
logger.info(f"正在加载模型: {self.config.model_name}")
|
||||
|
||||
try:
|
||||
# 构建模型参数
|
||||
model_kwargs = {}
|
||||
if self.config.cache_dir:
|
||||
model_kwargs["cache_folder"] = self.config.cache_dir
|
||||
|
||||
# 加载模型
|
||||
self._model = SentenceTransformer(
|
||||
self.config.model_path,
|
||||
**model_kwargs,
|
||||
)
|
||||
|
||||
logger.info(f"模型加载成功: {self.config.model_name}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"模型加载失败: {e}")
|
||||
raise
|
||||
|
||||
def encode(
|
||||
self,
|
||||
texts: Union[str, List[str]],
|
||||
batch_size: Optional[int] = None,
|
||||
show_progress: bool = False,
|
||||
normalize: bool = True,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
生成文本嵌入
|
||||
|
||||
Args:
|
||||
texts: 文本或文本列表
|
||||
batch_size: 批次大小（默认使用配置值）
|
||||
show_progress: 是否显示进度条
|
||||
normalize: 是否归一化
|
||||
|
||||
Returns:
|
||||
嵌入向量 (N x D)
|
||||
"""
|
||||
# 确保模型已加载
|
||||
self.load_model()
|
||||
|
||||
# 标准化输入
|
||||
if isinstance(texts, str):
|
||||
texts = [texts]
|
||||
single_input = True
|
||||
else:
|
||||
single_input = False
|
||||
|
||||
if not texts:
|
||||
return np.zeros((0, self.config.dimension), dtype=np.float32)
|
||||
|
||||
# 使用配置的批次大小
|
||||
if batch_size is None:
|
||||
batch_size = self.config.batch_size
|
||||
|
||||
# 生成嵌入
|
||||
try:
|
||||
embeddings = self._model.encode(
|
||||
texts,
|
||||
batch_size=batch_size,
|
||||
show_progress_bar=show_progress,
|
||||
normalize_embeddings=normalize and self.config.normalization,
|
||||
convert_to_numpy=True,
|
||||
)
|
||||
|
||||
# 确保是2D数组
|
||||
if embeddings.ndim == 1:
|
||||
embeddings = embeddings.reshape(1, -1)
|
||||
|
||||
self._total_encoded += len(texts)
|
||||
|
||||
# 如果是单个输入，返回1D数组
|
||||
if single_input:
|
||||
return embeddings[0]
|
||||
|
||||
return embeddings
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"生成嵌入失败: {e}")
|
||||
raise
|
||||
|
||||
def encode_batch(
|
||||
self,
|
||||
texts: List[str],
|
||||
batch_size: Optional[int] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
show_progress: bool = False,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
批量生成嵌入（多线程优化）
|
||||
|
||||
Args:
|
||||
texts: 文本列表
|
||||
batch_size: 批次大小
|
||||
num_workers: 工作线程数（默认使用初始化时的值）
|
||||
show_progress: 是否显示进度条
|
||||
|
||||
Returns:
|
||||
嵌入向量 (N x D)
|
||||
"""
|
||||
if not texts:
|
||||
return np.zeros((0, self.config.dimension), dtype=np.float32)
|
||||
|
||||
# 单线程模式
|
||||
num_workers = num_workers if num_workers is not None else self.num_workers
|
||||
if num_workers == 1:
|
||||
return self.encode(texts, batch_size=batch_size, show_progress=show_progress)
|
||||
|
||||
# 多线程模式
|
||||
logger.info(f"使用 {num_workers} 个线程生成 {len(texts)} 个嵌入")
|
||||
|
||||
# 分批
|
||||
batch_size = batch_size or self.config.batch_size
|
||||
batches = [
|
||||
texts[i:i + batch_size]
|
||||
for i in range(0, len(texts), batch_size)
|
||||
]
|
||||
|
||||
# 多线程生成
|
||||
all_embeddings = []
|
||||
with ThreadPoolExecutor(max_workers=num_workers) as executor:
|
||||
# 提交任务
|
||||
future_to_batch = {
|
||||
executor.submit(
|
||||
self.encode,
|
||||
batch,
|
||||
batch_size,
|
||||
False, # 不显示进度条（多线程时会混乱）
|
||||
): i
|
||||
for i, batch in enumerate(batches)
|
||||
}
|
||||
|
||||
# 收集结果
|
||||
for future in as_completed(future_to_batch):
|
||||
batch_idx = future_to_batch[future]
|
||||
try:
|
||||
embeddings = future.result()
|
||||
all_embeddings.append((batch_idx, embeddings))
|
||||
except Exception as e:
|
||||
logger.error(f"批次 {batch_idx} 生成嵌入失败: {e}")
|
||||
raise
|
||||
|
||||
# 按顺序合并
|
||||
all_embeddings.sort(key=lambda x: x[0])
|
||||
final_embeddings = np.concatenate([emb for _, emb in all_embeddings], axis=0)
|
||||
|
||||
return final_embeddings
|
||||
|
||||
def encode_with_cache(
|
||||
self,
|
||||
texts: List[str],
|
||||
batch_size: Optional[int] = None,
|
||||
show_progress: bool = False,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
生成嵌入（带缓存）
|
||||
|
||||
Args:
|
||||
texts: 文本列表
|
||||
batch_size: 批次大小
|
||||
show_progress: 是否显示进度条
|
||||
|
||||
Returns:
|
||||
嵌入向量 (N x D)
|
||||
"""
|
||||
if not self.enable_cache:
|
||||
return self.encode(texts, batch_size, show_progress)
|
||||
|
||||
# 分离缓存命中和未命中的文本
|
||||
cached_embeddings = []
|
||||
uncached_texts = []
|
||||
uncached_indices = []
|
||||
|
||||
for i, text in enumerate(texts):
|
||||
cache_key = self._get_cache_key(text)
|
||||
|
||||
with self._cache_lock:
|
||||
if cache_key in self._embedding_cache:
|
||||
cached_embeddings.append((i, self._embedding_cache[cache_key]))
|
||||
self._cache_hits += 1
|
||||
else:
|
||||
uncached_texts.append(text)
|
||||
uncached_indices.append(i)
|
||||
self._cache_misses += 1
|
||||
|
||||
# 生成未缓存的嵌入
|
||||
if uncached_texts:
|
||||
new_embeddings = self.encode(
|
||||
uncached_texts,
|
||||
batch_size,
|
||||
show_progress,
|
||||
)
|
||||
|
||||
# 更新缓存
|
||||
with self._cache_lock:
|
||||
for text, embedding in zip(uncached_texts, new_embeddings):
|
||||
cache_key = self._get_cache_key(text)
|
||||
self._embedding_cache[cache_key] = embedding.copy()
|
||||
|
||||
# 合并结果
|
||||
for idx, embedding in zip(uncached_indices, new_embeddings):
|
||||
cached_embeddings.append((idx, embedding))
|
||||
|
||||
# 按原始顺序排序
|
||||
cached_embeddings.sort(key=lambda x: x[0])
|
||||
final_embeddings = np.array([emb for _, emb in cached_embeddings])
|
||||
|
||||
return final_embeddings
|
||||
|
||||
def save_cache(self, cache_path: Optional[Union[str, Path]] = None) -> None:
|
||||
"""
|
||||
保存缓存到磁盘
|
||||
|
||||
Args:
|
||||
cache_path: 缓存文件路径（默认使用cache_dir/embeddings_cache.pkl）
|
||||
"""
|
||||
if cache_path is None:
|
||||
if self.cache_dir is None:
|
||||
raise ValueError("未指定缓存目录")
|
||||
cache_path = self.cache_dir / "embeddings_cache.pkl"
|
||||
|
||||
cache_path = Path(cache_path)
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with self._cache_lock:
|
||||
with open(cache_path, "wb") as f:
|
||||
pickle.dump(self._embedding_cache, f)
|
||||
|
||||
logger.info(f"缓存已保存: {cache_path} ({len(self._embedding_cache)} 条)")
|
||||
|
||||
def load_cache(self, cache_path: Optional[Union[str, Path]] = None) -> None:
|
||||
"""
|
||||
从磁盘加载缓存
|
||||
|
||||
Args:
|
||||
cache_path: 缓存文件路径（默认使用cache_dir/embeddings_cache.pkl）
|
||||
"""
|
||||
if cache_path is None:
|
||||
if self.cache_dir is None:
|
||||
raise ValueError("未指定缓存目录")
|
||||
cache_path = self.cache_dir / "embeddings_cache.pkl"
|
||||
|
||||
cache_path = Path(cache_path)
|
||||
if not cache_path.exists():
|
||||
logger.warning(f"缓存文件不存在: {cache_path}")
|
||||
return
|
||||
|
||||
with self._cache_lock:
|
||||
with open(cache_path, "rb") as f:
|
||||
self._embedding_cache = pickle.load(f)
|
||||
|
||||
logger.info(f"缓存已加载: {cache_path} ({len(self._embedding_cache)} 条)")
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""清空缓存"""
|
||||
with self._cache_lock:
|
||||
count = len(self._embedding_cache)
|
||||
self._embedding_cache.clear()
|
||||
logger.info(f"已清空缓存: {count} 条")
|
||||
|
||||
def check_model_consistency(
|
||||
self,
|
||||
stored_embeddings: np.ndarray,
|
||||
sample_texts: List[str] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
"""
|
||||
检查模型一致性
|
||||
|
||||
Args:
|
||||
stored_embeddings: 存储的嵌入向量
|
||||
sample_texts: 样本文本（用于重新生成对比）
|
||||
|
||||
Returns:
|
||||
(是否一致, 详细信息)
|
||||
"""
|
||||
# 检查维度
|
||||
if stored_embeddings.shape[1] != self.config.dimension:
|
||||
return False, f"维度不匹配: 期望 {self.config.dimension}, 实际 {stored_embeddings.shape[1]}"
|
||||
|
||||
# 如果提供了样本文本，重新生成并比较
|
||||
if sample_texts:
|
||||
try:
|
||||
new_embeddings = self.encode(sample_texts[:5]) # 只比较前5个
|
||||
|
||||
# 计算相似度
|
||||
similarities = np.dot(
|
||||
stored_embeddings[:5],
|
||||
new_embeddings.T,
|
||||
).diagonal()
|
||||
|
||||
# 检查相似度
|
||||
if np.mean(similarities) < 0.95:
|
||||
return False, f"模型可能已更改，平均相似度: {np.mean(similarities):.3f}"
|
||||
|
||||
return True, f"模型一致，平均相似度: {np.mean(similarities):.3f}"
|
||||
|
||||
except Exception as e:
|
||||
return False, f"一致性检查失败: {e}"
|
||||
|
||||
return True, "维度匹配"
|
||||
|
||||
def get_model_info(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取模型信息
|
||||
|
||||
Returns:
|
||||
模型信息字典
|
||||
"""
|
||||
return {
|
||||
"model_name": self.config.model_name,
|
||||
"dimension": self.config.dimension,
|
||||
"max_seq_length": self.config.max_seq_length,
|
||||
"batch_size": self.config.batch_size,
|
||||
"normalization": self.config.normalization,
|
||||
"pooling": self.config.pooling,
|
||||
"model_loaded": self._model is not None,
|
||||
"cache_enabled": self.enable_cache,
|
||||
"cache_size": len(self._embedding_cache),
|
||||
"total_encoded": self._total_encoded,
|
||||
"cache_hits": self._cache_hits,
|
||||
"cache_misses": self._cache_misses,
|
||||
}
|
||||
|
||||
def get_embedding_dimension(self) -> int:
|
||||
"""获取嵌入维度"""
|
||||
return self.config.dimension
|
||||
|
||||
def _get_cache_key(self, text: str) -> str:
|
||||
"""
|
||||
生成缓存键
|
||||
|
||||
Args:
|
||||
text: 文本内容
|
||||
|
||||
Returns:
|
||||
缓存键（SHA256哈希）
|
||||
"""
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
@property
|
||||
def is_model_loaded(self) -> bool:
|
||||
"""模型是否已加载"""
|
||||
return self._model is not None
|
||||
|
||||
@property
|
||||
def cache_hit_rate(self) -> float:
|
||||
"""缓存命中率"""
|
||||
total = self._cache_hits + self._cache_misses
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return self._cache_hits / total
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"EmbeddingManager(model={self.config.model_name}, "
|
||||
f"dim={self.config.dimension}, "
|
||||
f"loaded={self.is_model_loaded}, "
|
||||
f"cache={len(self._embedding_cache)})"
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
def create_embedding_manager_from_config(
|
||||
model_name: str,
|
||||
model_path: str,
|
||||
dimension: int,
|
||||
cache_dir: Optional[Union[str, Path]] = None,
|
||||
enable_cache: bool = True,
|
||||
num_workers: int = 1,
|
||||
**config_kwargs,
|
||||
) -> EmbeddingManager:
|
||||
"""
|
||||
从自定义配置创建嵌入管理器
|
||||
|
||||
Args:
|
||||
model_name: 模型名称
|
||||
model_path: HuggingFace模型路径
|
||||
dimension: 输出维度
|
||||
cache_dir: 缓存目录
|
||||
enable_cache: 是否启用缓存
|
||||
num_workers: 工作线程数
|
||||
**config_kwargs: 其他配置参数
|
||||
|
||||
Returns:
|
||||
嵌入管理器实例
|
||||
"""
|
||||
# 创建自定义配置
|
||||
config = get_custom_config(
|
||||
model_name=model_name,
|
||||
model_path=model_path,
|
||||
dimension=dimension,
|
||||
cache_dir=cache_dir,
|
||||
**config_kwargs,
|
||||
)
|
||||
|
||||
# 创建管理器
|
||||
return EmbeddingManager(
|
||||
config=config,
|
||||
cache_dir=cache_dir,
|
||||
enable_cache=enable_cache,
|
||||
num_workers=num_workers,
|
||||
)
|
||||
72
src/A_memorix/core/embedding/presets.py
Normal file
72
src/A_memorix/core/embedding/presets.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
嵌入模型配置模块
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Dict, Any, Union
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmbeddingModelConfig:
|
||||
"""
|
||||
嵌入模型配置
|
||||
|
||||
属性：
|
||||
model_name: 模型描述名称
|
||||
model_path: 实际加载路径（Local or HF）
|
||||
dimension: 嵌入向量维度
|
||||
max_seq_length: 最大序列长度
|
||||
batch_size: 编码批次大小
|
||||
model_size_mb: 估计显存占用
|
||||
description: 模型说明
|
||||
normalization: 是否自动归一化
|
||||
pooling: 池化策略 (mean, cls, max)
|
||||
cache_dir: 模型缓存目录
|
||||
"""
|
||||
|
||||
model_name: str
|
||||
model_path: str
|
||||
dimension: int
|
||||
max_seq_length: int = 512
|
||||
batch_size: int = 32
|
||||
model_size_mb: int = 100
|
||||
description: str = ""
|
||||
normalization: bool = True
|
||||
pooling: str = "mean"
|
||||
cache_dir: Optional[Union[str, Path]] = None
|
||||
|
||||
|
||||
def validate_config_compatibility(
|
||||
config1: EmbeddingModelConfig, config2: EmbeddingModelConfig
|
||||
) -> bool:
|
||||
"""检查两个配置是否兼容（主要看维度）"""
|
||||
return config1.dimension == config2.dimension
|
||||
|
||||
|
||||
def are_models_compatible(
|
||||
config1: EmbeddingModelConfig, config2: EmbeddingModelConfig
|
||||
) -> bool:
|
||||
"""检查模型是否完全相同（用于热切换判断）"""
|
||||
return (
|
||||
config1.model_path == config2.model_path
|
||||
and config1.dimension == config2.dimension
|
||||
and config1.pooling == config2.pooling
|
||||
)
|
||||
|
||||
|
||||
def get_custom_config(
|
||||
model_name: str,
|
||||
model_path: str,
|
||||
dimension: int,
|
||||
cache_dir: Optional[Union[str, Path]] = None,
|
||||
**kwargs,
|
||||
) -> EmbeddingModelConfig:
|
||||
"""创建自定义模型配置"""
|
||||
return EmbeddingModelConfig(
|
||||
model_name=model_name,
|
||||
model_path=model_path,
|
||||
dimension=dimension,
|
||||
cache_dir=cache_dir,
|
||||
**kwargs,
|
||||
)
|
||||
54
src/A_memorix/core/retrieval/__init__.py
Normal file
54
src/A_memorix/core/retrieval/__init__.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""检索模块 - 双路检索与排序"""
|
||||
|
||||
from .dual_path import (
|
||||
DualPathRetriever,
|
||||
RetrievalStrategy,
|
||||
RetrievalResult,
|
||||
DualPathRetrieverConfig,
|
||||
TemporalQueryOptions,
|
||||
FusionConfig,
|
||||
RelationIntentConfig,
|
||||
)
|
||||
from .pagerank import (
|
||||
PersonalizedPageRank,
|
||||
PageRankConfig,
|
||||
create_ppr_from_graph,
|
||||
)
|
||||
from .threshold import (
|
||||
DynamicThresholdFilter,
|
||||
ThresholdMethod,
|
||||
ThresholdConfig,
|
||||
)
|
||||
from .sparse_bm25 import (
|
||||
SparseBM25Index,
|
||||
SparseBM25Config,
|
||||
)
|
||||
from .graph_relation_recall import (
|
||||
GraphRelationRecallConfig,
|
||||
GraphRelationRecallService,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# DualPathRetriever
|
||||
"DualPathRetriever",
|
||||
"RetrievalStrategy",
|
||||
"RetrievalResult",
|
||||
"DualPathRetrieverConfig",
|
||||
"TemporalQueryOptions",
|
||||
"FusionConfig",
|
||||
"RelationIntentConfig",
|
||||
# PersonalizedPageRank
|
||||
"PersonalizedPageRank",
|
||||
"PageRankConfig",
|
||||
"create_ppr_from_graph",
|
||||
# DynamicThresholdFilter
|
||||
"DynamicThresholdFilter",
|
||||
"ThresholdMethod",
|
||||
"ThresholdConfig",
|
||||
# Sparse BM25
|
||||
"SparseBM25Index",
|
||||
"SparseBM25Config",
|
||||
# Graph relation recall
|
||||
"GraphRelationRecallConfig",
|
||||
"GraphRelationRecallService",
|
||||
]
|
||||
1871
src/A_memorix/core/retrieval/dual_path.py
Normal file
1871
src/A_memorix/core/retrieval/dual_path.py
Normal file
File diff suppressed because it is too large
Load Diff
272
src/A_memorix/core/retrieval/graph_relation_recall.py
Normal file
272
src/A_memorix/core/retrieval/graph_relation_recall.py
Normal file
@@ -0,0 +1,272 @@
|
||||
"""Graph-assisted relation candidate recall for relation-oriented queries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Sequence, Set
|
||||
|
||||
from src.common.logger import get_logger
|
||||
|
||||
logger = get_logger("A_Memorix.GraphRelationRecall")
|
||||
|
||||
|
||||
@dataclass
|
||||
class GraphRelationRecallConfig:
|
||||
"""Configuration for controlled graph relation recall."""
|
||||
|
||||
enabled: bool = True
|
||||
candidate_k: int = 24
|
||||
max_hop: int = 1
|
||||
allow_two_hop_pair: bool = True
|
||||
max_paths: int = 4
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.enabled = bool(self.enabled)
|
||||
self.candidate_k = max(1, int(self.candidate_k))
|
||||
self.max_hop = max(1, int(self.max_hop))
|
||||
self.allow_two_hop_pair = bool(self.allow_two_hop_pair)
|
||||
self.max_paths = max(1, int(self.max_paths))
|
||||
|
||||
|
||||
@dataclass
|
||||
class GraphRelationCandidate:
|
||||
"""A graph-derived relation candidate before retriever-side fusion."""
|
||||
|
||||
hash_value: str
|
||||
subject: str
|
||||
predicate: str
|
||||
object: str
|
||||
confidence: float
|
||||
graph_seed_entities: List[str]
|
||||
graph_hops: int
|
||||
graph_candidate_type: str
|
||||
supporting_paragraph_count: int
|
||||
|
||||
def to_payload(self) -> Dict[str, Any]:
|
||||
content = f"{self.subject} {self.predicate} {self.object}"
|
||||
return {
|
||||
"hash": self.hash_value,
|
||||
"content": content,
|
||||
"subject": self.subject,
|
||||
"predicate": self.predicate,
|
||||
"object": self.object,
|
||||
"confidence": self.confidence,
|
||||
"graph_seed_entities": list(self.graph_seed_entities),
|
||||
"graph_hops": int(self.graph_hops),
|
||||
"graph_candidate_type": self.graph_candidate_type,
|
||||
"supporting_paragraph_count": int(self.supporting_paragraph_count),
|
||||
}
|
||||
|
||||
|
||||
class GraphRelationRecallService:
|
||||
"""Collect relation candidates from the entity graph in a controlled way."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
graph_store: Any,
|
||||
metadata_store: Any,
|
||||
config: Optional[GraphRelationRecallConfig] = None,
|
||||
) -> None:
|
||||
self.graph_store = graph_store
|
||||
self.metadata_store = metadata_store
|
||||
self.config = config or GraphRelationRecallConfig()
|
||||
|
||||
def recall(
|
||||
self,
|
||||
*,
|
||||
seed_entities: Sequence[str],
|
||||
) -> List[GraphRelationCandidate]:
|
||||
if not self.config.enabled:
|
||||
return []
|
||||
if self.graph_store is None or self.metadata_store is None:
|
||||
return []
|
||||
|
||||
seeds = self._normalize_seed_entities(seed_entities)
|
||||
if not seeds:
|
||||
return []
|
||||
|
||||
seen_hashes: Set[str] = set()
|
||||
candidates: List[GraphRelationCandidate] = []
|
||||
|
||||
if len(seeds) >= 2:
|
||||
self._collect_direct_pair_candidates(
|
||||
seed_a=seeds[0],
|
||||
seed_b=seeds[1],
|
||||
seen_hashes=seen_hashes,
|
||||
out=candidates,
|
||||
)
|
||||
if (
|
||||
len(candidates) < 3
|
||||
and self.config.allow_two_hop_pair
|
||||
and len(candidates) < self.config.candidate_k
|
||||
):
|
||||
self._collect_two_hop_pair_candidates(
|
||||
seed_a=seeds[0],
|
||||
seed_b=seeds[1],
|
||||
seen_hashes=seen_hashes,
|
||||
out=candidates,
|
||||
)
|
||||
else:
|
||||
self._collect_one_hop_seed_candidates(
|
||||
seed=seeds[0],
|
||||
seen_hashes=seen_hashes,
|
||||
out=candidates,
|
||||
)
|
||||
|
||||
return candidates[: self.config.candidate_k]
|
||||
|
||||
def _normalize_seed_entities(self, seed_entities: Sequence[str]) -> List[str]:
|
||||
out: List[str] = []
|
||||
seen = set()
|
||||
for raw in list(seed_entities)[:2]:
|
||||
resolved = None
|
||||
try:
|
||||
resolved = self.graph_store.find_node(str(raw), ignore_case=True)
|
||||
except Exception:
|
||||
resolved = None
|
||||
if not resolved:
|
||||
continue
|
||||
canon = str(resolved).strip().lower()
|
||||
if not canon or canon in seen:
|
||||
continue
|
||||
seen.add(canon)
|
||||
out.append(str(resolved))
|
||||
return out
|
||||
|
||||
def _collect_direct_pair_candidates(
|
||||
self,
|
||||
*,
|
||||
seed_a: str,
|
||||
seed_b: str,
|
||||
seen_hashes: Set[str],
|
||||
out: List[GraphRelationCandidate],
|
||||
) -> None:
|
||||
relation_hashes = []
|
||||
relation_hashes.extend(self.graph_store.get_relation_hashes_for_edge(seed_a, seed_b))
|
||||
relation_hashes.extend(self.graph_store.get_relation_hashes_for_edge(seed_b, seed_a))
|
||||
self._append_relation_hashes(
|
||||
relation_hashes=relation_hashes,
|
||||
seen_hashes=seen_hashes,
|
||||
out=out,
|
||||
candidate_type="direct_pair",
|
||||
graph_hops=1,
|
||||
graph_seed_entities=[seed_a, seed_b],
|
||||
)
|
||||
|
||||
def _collect_two_hop_pair_candidates(
|
||||
self,
|
||||
*,
|
||||
seed_a: str,
|
||||
seed_b: str,
|
||||
seen_hashes: Set[str],
|
||||
out: List[GraphRelationCandidate],
|
||||
) -> None:
|
||||
try:
|
||||
paths = self.graph_store.find_paths(
|
||||
seed_a,
|
||||
seed_b,
|
||||
max_depth=2,
|
||||
max_paths=self.config.max_paths,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"graph two-hop recall skipped: {e}")
|
||||
return
|
||||
|
||||
for path_nodes in paths:
|
||||
if len(out) >= self.config.candidate_k:
|
||||
break
|
||||
if not isinstance(path_nodes, Sequence) or len(path_nodes) < 3:
|
||||
continue
|
||||
if len(path_nodes) != 3:
|
||||
continue
|
||||
for idx in range(len(path_nodes) - 1):
|
||||
if len(out) >= self.config.candidate_k:
|
||||
break
|
||||
u = str(path_nodes[idx])
|
||||
v = str(path_nodes[idx + 1])
|
||||
relation_hashes = []
|
||||
relation_hashes.extend(self.graph_store.get_relation_hashes_for_edge(u, v))
|
||||
relation_hashes.extend(self.graph_store.get_relation_hashes_for_edge(v, u))
|
||||
self._append_relation_hashes(
|
||||
relation_hashes=relation_hashes,
|
||||
seen_hashes=seen_hashes,
|
||||
out=out,
|
||||
candidate_type="two_hop_pair",
|
||||
graph_hops=2,
|
||||
graph_seed_entities=[seed_a, seed_b],
|
||||
)
|
||||
|
||||
def _collect_one_hop_seed_candidates(
|
||||
self,
|
||||
*,
|
||||
seed: str,
|
||||
seen_hashes: Set[str],
|
||||
out: List[GraphRelationCandidate],
|
||||
) -> None:
|
||||
try:
|
||||
relation_hashes = self.graph_store.get_incident_relation_hashes(
|
||||
seed,
|
||||
limit=self.config.candidate_k,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"graph one-hop recall skipped: {e}")
|
||||
return
|
||||
self._append_relation_hashes(
|
||||
relation_hashes=relation_hashes,
|
||||
seen_hashes=seen_hashes,
|
||||
out=out,
|
||||
candidate_type="one_hop_seed",
|
||||
graph_hops=min(1, self.config.max_hop),
|
||||
graph_seed_entities=[seed],
|
||||
)
|
||||
|
||||
def _append_relation_hashes(
|
||||
self,
|
||||
*,
|
||||
relation_hashes: Sequence[str],
|
||||
seen_hashes: Set[str],
|
||||
out: List[GraphRelationCandidate],
|
||||
candidate_type: str,
|
||||
graph_hops: int,
|
||||
graph_seed_entities: Sequence[str],
|
||||
) -> None:
|
||||
for relation_hash in sorted({str(h) for h in relation_hashes if str(h).strip()}):
|
||||
if len(out) >= self.config.candidate_k:
|
||||
break
|
||||
if relation_hash in seen_hashes:
|
||||
continue
|
||||
candidate = self._build_candidate(
|
||||
relation_hash=relation_hash,
|
||||
candidate_type=candidate_type,
|
||||
graph_hops=graph_hops,
|
||||
graph_seed_entities=graph_seed_entities,
|
||||
)
|
||||
if candidate is None:
|
||||
continue
|
||||
seen_hashes.add(relation_hash)
|
||||
out.append(candidate)
|
||||
|
||||
def _build_candidate(
|
||||
self,
|
||||
*,
|
||||
relation_hash: str,
|
||||
candidate_type: str,
|
||||
graph_hops: int,
|
||||
graph_seed_entities: Sequence[str],
|
||||
) -> Optional[GraphRelationCandidate]:
|
||||
relation = self.metadata_store.get_relation(relation_hash)
|
||||
if relation is None:
|
||||
return None
|
||||
supporting_paragraphs = self.metadata_store.get_paragraphs_by_relation(relation_hash)
|
||||
return GraphRelationCandidate(
|
||||
hash_value=relation_hash,
|
||||
subject=str(relation.get("subject", "")),
|
||||
predicate=str(relation.get("predicate", "")),
|
||||
object=str(relation.get("object", "")),
|
||||
confidence=float(relation.get("confidence", 1.0) or 1.0),
|
||||
graph_seed_entities=[str(x) for x in graph_seed_entities],
|
||||
graph_hops=int(graph_hops),
|
||||
graph_candidate_type=str(candidate_type),
|
||||
supporting_paragraph_count=len(supporting_paragraphs),
|
||||
)
|
||||
482
src/A_memorix/core/retrieval/pagerank.py
Normal file
482
src/A_memorix/core/retrieval/pagerank.py
Normal file
@@ -0,0 +1,482 @@
|
||||
"""
|
||||
Personalized PageRank实现
|
||||
|
||||
提供个性化的图节点排序功能。
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional, Tuple, Union, Any
|
||||
from dataclasses import dataclass
|
||||
import numpy as np
|
||||
|
||||
from src.common.logger import get_logger
|
||||
from ..storage import GraphStore
|
||||
from ..utils.matcher import AhoCorasick
|
||||
|
||||
logger = get_logger("A_Memorix.PersonalizedPageRank")
|
||||
|
||||
|
||||
@dataclass
|
||||
class PageRankConfig:
|
||||
"""
|
||||
PageRank配置
|
||||
|
||||
属性：
|
||||
alpha: 阻尼系数（0-1之间）
|
||||
max_iter: 最大迭代次数
|
||||
tol: 收敛阈值
|
||||
normalize: 是否归一化结果
|
||||
min_iterations: 最小迭代次数
|
||||
"""
|
||||
|
||||
alpha: float = 0.85
|
||||
max_iter: int = 100
|
||||
tol: float = 1e-6
|
||||
normalize: bool = True
|
||||
min_iterations: int = 20
|
||||
|
||||
def __post_init__(self):
|
||||
"""验证配置"""
|
||||
if not 0 <= self.alpha < 1:
|
||||
raise ValueError(f"alpha必须在[0, 1)之间: {self.alpha}")
|
||||
|
||||
if self.max_iter <= 0:
|
||||
raise ValueError(f"max_iter必须大于0: {self.max_iter}")
|
||||
|
||||
if self.tol <= 0:
|
||||
raise ValueError(f"tol必须大于0: {self.tol}")
|
||||
|
||||
if self.min_iterations < 0:
|
||||
raise ValueError(f"min_iterations必须大于等于0: {self.min_iterations}")
|
||||
|
||||
if self.min_iterations >= self.max_iter:
|
||||
raise ValueError(f"min_iterations必须小于max_iter")
|
||||
|
||||
|
||||
class PersonalizedPageRank:
|
||||
"""
|
||||
Personalized PageRank计算器
|
||||
|
||||
功能：
|
||||
- 个性化向量支持
|
||||
- 快速收敛检测
|
||||
- 结果归一化
|
||||
- 批量计算
|
||||
- 统计信息
|
||||
|
||||
参数：
|
||||
graph_store: 图存储
|
||||
config: PageRank配置
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph_store: GraphStore,
|
||||
config: Optional[PageRankConfig] = None,
|
||||
):
|
||||
"""
|
||||
初始化PPR计算器
|
||||
|
||||
Args:
|
||||
graph_store: 图存储
|
||||
config: PageRank配置
|
||||
"""
|
||||
self.graph_store = graph_store
|
||||
self.config = config or PageRankConfig()
|
||||
|
||||
# 统计信息
|
||||
self._total_computations = 0
|
||||
self._total_iterations = 0
|
||||
self._convergence_history: List[int] = []
|
||||
|
||||
logger.info(
|
||||
f"PersonalizedPageRank 初始化: "
|
||||
f"alpha={self.config.alpha}, "
|
||||
f"max_iter={self.config.max_iter}"
|
||||
)
|
||||
|
||||
# 缓存 Aho-Corasick 匹配器
|
||||
self._ac_matcher: Optional[AhoCorasick] = None
|
||||
self._ac_nodes_count = 0
|
||||
|
||||
def compute(
|
||||
self,
|
||||
personalization: Optional[Dict[str, float]] = None,
|
||||
alpha: Optional[float] = None,
|
||||
max_iter: Optional[int] = None,
|
||||
normalize: Optional[bool] = None,
|
||||
) -> Dict[str, float]:
|
||||
"""
|
||||
计算Personalized PageRank
|
||||
|
||||
Args:
|
||||
personalization: 个性化向量 {节点名: 权重}
|
||||
alpha: 阻尼系数（覆盖配置值）
|
||||
max_iter: 最大迭代次数（覆盖配置值）
|
||||
normalize: 是否归一化（覆盖配置值）
|
||||
|
||||
Returns:
|
||||
节点PageRank值字典 {节点名: 分数}
|
||||
"""
|
||||
# 使用覆盖值或配置值
|
||||
alpha = alpha if alpha is not None else self.config.alpha
|
||||
max_iter = max_iter if max_iter is not None else self.config.max_iter
|
||||
normalize = normalize if normalize is not None else self.config.normalize
|
||||
|
||||
# 调用GraphStore的compute_pagerank
|
||||
scores = self.graph_store.compute_pagerank(
|
||||
personalization=personalization,
|
||||
alpha=alpha,
|
||||
max_iter=max_iter,
|
||||
tol=self.config.tol,
|
||||
)
|
||||
|
||||
# 归一化（如果需要）
|
||||
if normalize and scores:
|
||||
total = sum(scores.values())
|
||||
if total > 0:
|
||||
scores = {node: score / total for node, score in scores.items()}
|
||||
|
||||
# 更新统计
|
||||
self._total_computations += 1
|
||||
|
||||
logger.debug(
|
||||
f"PPR计算完成: {len(scores)} 个节点, "
|
||||
f"personalization_nodes={len(personalization) if personalization else 0}"
|
||||
)
|
||||
|
||||
return scores
|
||||
|
||||
def compute_batch(
|
||||
self,
|
||||
personalization_list: List[Dict[str, float]],
|
||||
normalize: bool = True,
|
||||
) -> List[Dict[str, float]]:
|
||||
"""
|
||||
批量计算PPR
|
||||
|
||||
Args:
|
||||
personalization_list: 个性化向量列表
|
||||
normalize: 是否归一化
|
||||
|
||||
Returns:
|
||||
PageRank值字典列表
|
||||
"""
|
||||
results = []
|
||||
|
||||
for i, personalization in enumerate(personalization_list):
|
||||
logger.debug(f"计算第 {i+1}/{len(personalization_list)} 个PPR")
|
||||
scores = self.compute(personalization=personalization, normalize=normalize)
|
||||
results.append(scores)
|
||||
|
||||
return results
|
||||
|
||||
def compute_for_entities(
|
||||
self,
|
||||
entities: List[str],
|
||||
weights: Optional[List[float]] = None,
|
||||
normalize: bool = True,
|
||||
) -> Dict[str, float]:
|
||||
"""
|
||||
为实体列表计算PPR
|
||||
|
||||
Args:
|
||||
entities: 实体列表
|
||||
weights: 权重列表（默认均匀权重）
|
||||
normalize: 是否归一化
|
||||
|
||||
Returns:
|
||||
PageRank值字典
|
||||
"""
|
||||
if not entities:
|
||||
logger.warning("实体列表为空，返回均匀PPR")
|
||||
return self.compute(personalization=None, normalize=normalize)
|
||||
|
||||
# 构建个性化向量
|
||||
if weights is None:
|
||||
weights = [1.0] * len(entities)
|
||||
|
||||
if len(weights) != len(entities):
|
||||
raise ValueError(f"权重数量与实体数量不匹配: {len(weights)} vs {len(entities)}")
|
||||
|
||||
personalization = {entity: weight for entity, weight in zip(entities, weights)}
|
||||
|
||||
return self.compute(personalization=personalization, normalize=normalize)
|
||||
|
||||
def compute_for_query(
|
||||
self,
|
||||
query: str,
|
||||
entity_extractor: Optional[callable] = None,
|
||||
normalize: bool = True,
|
||||
) -> Dict[str, float]:
|
||||
"""
|
||||
为查询计算PPR
|
||||
|
||||
Args:
|
||||
query: 查询文本
|
||||
entity_extractor: 实体提取函数（可选）
|
||||
normalize: 是否归一化
|
||||
|
||||
Returns:
|
||||
PageRank值字典
|
||||
"""
|
||||
# 提取实体
|
||||
if entity_extractor is not None:
|
||||
entities = entity_extractor(query)
|
||||
else:
|
||||
# 简单实现：基于图中的节点匹配
|
||||
entities = self._extract_entities_from_query(query)
|
||||
|
||||
if not entities:
|
||||
logger.debug(f"未从查询中提取到实体: '{query}'")
|
||||
return self.compute(personalization=None, normalize=normalize)
|
||||
|
||||
# 计算PPR
|
||||
return self.compute_for_entities(entities, normalize=normalize)
|
||||
|
||||
def rank_nodes(
|
||||
self,
|
||||
scores: Dict[str, float],
|
||||
top_k: Optional[int] = None,
|
||||
min_score: float = 0.0,
|
||||
) -> List[Tuple[str, float]]:
|
||||
"""
|
||||
对节点排序
|
||||
|
||||
Args:
|
||||
scores: PageRank分数字典
|
||||
top_k: 返回前k个节点（None表示全部）
|
||||
min_score: 最小分数阈值
|
||||
|
||||
Returns:
|
||||
排序后的节点列表 [(节点名, 分数), ...]
|
||||
"""
|
||||
# 过滤低分节点
|
||||
filtered = [(node, score) for node, score in scores.items() if score >= min_score]
|
||||
|
||||
# 按分数降序排序
|
||||
sorted_nodes = sorted(filtered, key=lambda x: x[1], reverse=True)
|
||||
|
||||
# 返回top_k
|
||||
if top_k is not None:
|
||||
sorted_nodes = sorted_nodes[:top_k]
|
||||
|
||||
return sorted_nodes
|
||||
|
||||
def get_personalization_vector(
|
||||
self,
|
||||
nodes: List[str],
|
||||
method: str = "uniform",
|
||||
) -> Dict[str, float]:
|
||||
"""
|
||||
生成个性化向量
|
||||
|
||||
Args:
|
||||
nodes: 节点列表
|
||||
method: 生成方法
|
||||
- "uniform": 均匀权重
|
||||
- "degree": 按度数加权
|
||||
- "inverse_degree": 按度数反比加权
|
||||
|
||||
Returns:
|
||||
个性化向量 {节点名: 权重}
|
||||
"""
|
||||
if not nodes:
|
||||
return {}
|
||||
|
||||
if method == "uniform":
|
||||
# 均匀权重
|
||||
weight = 1.0 / len(nodes)
|
||||
return {node: weight for node in nodes}
|
||||
|
||||
elif method == "degree":
|
||||
# 按度数加权
|
||||
node_degrees = {}
|
||||
for node in nodes:
|
||||
neighbors = self.graph_store.get_neighbors(node)
|
||||
node_degrees[node] = len(neighbors)
|
||||
|
||||
total_degree = sum(node_degrees.values())
|
||||
if total_degree > 0:
|
||||
return {node: degree / total_degree for node, degree in node_degrees.items()}
|
||||
else:
|
||||
return {node: 1.0 / len(nodes) for node in nodes}
|
||||
|
||||
elif method == "inverse_degree":
|
||||
# 按度数反比加权
|
||||
node_degrees = {}
|
||||
for node in nodes:
|
||||
neighbors = self.graph_store.get_neighbors(node)
|
||||
node_degrees[node] = len(neighbors)
|
||||
|
||||
# 反度数
|
||||
inv_degrees = {node: 1.0 / (degree + 1) for node, degree in node_degrees.items()}
|
||||
total_inv = sum(inv_degrees.values())
|
||||
|
||||
if total_inv > 0:
|
||||
return {node: inv / total_inv for node, inv in inv_degrees.items()}
|
||||
else:
|
||||
return {node: 1.0 / len(nodes) for node in nodes}
|
||||
|
||||
else:
|
||||
raise ValueError(f"不支持的个性化向量生成方法: {method}")
|
||||
|
||||
def compare_scores(
|
||||
self,
|
||||
scores1: Dict[str, float],
|
||||
scores2: Dict[str, float],
|
||||
) -> Dict[str, Dict[str, float]]:
|
||||
"""
|
||||
比较两组PPR分数
|
||||
|
||||
Args:
|
||||
scores1: 第一组分数
|
||||
scores2: 第二组分数
|
||||
|
||||
Returns:
|
||||
比较结果 {
|
||||
"common_nodes": {节点: (score1, score2)},
|
||||
"only_in_1": {节点: score1},
|
||||
"only_in_2": {节点: score2},
|
||||
}
|
||||
"""
|
||||
common_nodes = {}
|
||||
only_in_1 = {}
|
||||
only_in_2 = {}
|
||||
|
||||
all_nodes = set(scores1.keys()) | set(scores2.keys())
|
||||
|
||||
for node in all_nodes:
|
||||
if node in scores1 and node in scores2:
|
||||
common_nodes[node] = (scores1[node], scores2[node])
|
||||
elif node in scores1:
|
||||
only_in_1[node] = scores1[node]
|
||||
else:
|
||||
only_in_2[node] = scores2[node]
|
||||
|
||||
return {
|
||||
"common_nodes": common_nodes,
|
||||
"only_in_1": only_in_1,
|
||||
"only_in_2": only_in_2,
|
||||
}
|
||||
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取统计信息
|
||||
|
||||
Returns:
|
||||
统计信息字典
|
||||
"""
|
||||
avg_iterations = (
|
||||
self._total_iterations / self._total_computations
|
||||
if self._total_computations > 0
|
||||
else 0
|
||||
)
|
||||
|
||||
return {
|
||||
"config": {
|
||||
"alpha": self.config.alpha,
|
||||
"max_iter": self.config.max_iter,
|
||||
"tol": self.config.tol,
|
||||
"normalize": self.config.normalize,
|
||||
"min_iterations": self.config.min_iterations,
|
||||
},
|
||||
"statistics": {
|
||||
"total_computations": self._total_computations,
|
||||
"total_iterations": self._total_iterations,
|
||||
"avg_iterations": avg_iterations,
|
||||
"convergence_history": self._convergence_history.copy(),
|
||||
},
|
||||
"graph": {
|
||||
"num_nodes": self.graph_store.num_nodes,
|
||||
"num_edges": self.graph_store.num_edges,
|
||||
},
|
||||
}
|
||||
|
||||
def reset_statistics(self) -> None:
|
||||
"""重置统计信息"""
|
||||
self._total_computations = 0
|
||||
self._total_iterations = 0
|
||||
self._convergence_history.clear()
|
||||
logger.info("统计信息已重置")
|
||||
|
||||
def _extract_entities_from_query(self, query: str) -> List[str]:
|
||||
"""
|
||||
从查询中提取实体（简化实现）
|
||||
|
||||
Args:
|
||||
query: 查询文本
|
||||
|
||||
Returns:
|
||||
实体列表
|
||||
"""
|
||||
# 获取所有节点
|
||||
all_nodes = self.graph_store.get_nodes()
|
||||
if not all_nodes:
|
||||
return []
|
||||
|
||||
# 检查是否需要更新 Aho-Corasick 匹配器
|
||||
if self._ac_matcher is None or self._ac_nodes_count != len(all_nodes):
|
||||
self._ac_matcher = AhoCorasick()
|
||||
for node in all_nodes:
|
||||
# 统一转为小写进行不区分大小写匹配
|
||||
self._ac_matcher.add_pattern(node.lower())
|
||||
self._ac_matcher.build()
|
||||
self._ac_nodes_count = len(all_nodes)
|
||||
|
||||
# 执行匹配
|
||||
query_lower = query.lower()
|
||||
stats = self._ac_matcher.find_all(query_lower)
|
||||
|
||||
# 转换回原始的大小写（这里简化为从 all_nodes 中找，或者 AC 存原始值）
|
||||
# 为了简单，AC 中 add_pattern 存的是小写
|
||||
# 我们需要一个映射：小写 -> 原始
|
||||
node_map = {node.lower(): node for node in all_nodes}
|
||||
entities = [node_map[low_name] for low_name in stats.keys()]
|
||||
|
||||
return entities
|
||||
|
||||
@property
|
||||
def num_computations(self) -> int:
|
||||
"""计算次数"""
|
||||
return self._total_computations
|
||||
|
||||
@property
|
||||
def avg_iterations(self) -> float:
|
||||
"""平均迭代次数"""
|
||||
if self._total_computations == 0:
|
||||
return 0.0
|
||||
return self._total_iterations / self._total_computations
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"PersonalizedPageRank("
|
||||
f"alpha={self.config.alpha}, "
|
||||
f"computations={self._total_computations})"
|
||||
)
|
||||
|
||||
|
||||
def create_ppr_from_graph(
|
||||
graph_store: GraphStore,
|
||||
alpha: float = 0.85,
|
||||
max_iter: int = 100,
|
||||
) -> PersonalizedPageRank:
|
||||
"""
|
||||
从图存储创建PPR计算器
|
||||
|
||||
Args:
|
||||
graph_store: 图存储
|
||||
alpha: 阻尼系数
|
||||
max_iter: 最大迭代次数
|
||||
|
||||
Returns:
|
||||
PPR计算器实例
|
||||
"""
|
||||
config = PageRankConfig(
|
||||
alpha=alpha,
|
||||
max_iter=max_iter,
|
||||
)
|
||||
|
||||
return PersonalizedPageRank(
|
||||
graph_store=graph_store,
|
||||
config=config,
|
||||
)
|
||||
401
src/A_memorix/core/retrieval/sparse_bm25.py
Normal file
401
src/A_memorix/core/retrieval/sparse_bm25.py
Normal file
@@ -0,0 +1,401 @@
|
||||
"""
|
||||
稀疏检索组件（FTS5 + BM25）
|
||||
|
||||
支持：
|
||||
- 懒加载索引连接
|
||||
- jieba / char n-gram 分词
|
||||
- 可卸载并收缩 SQLite 内存缓存
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from src.common.logger import get_logger
|
||||
from ..storage import MetadataStore
|
||||
|
||||
logger = get_logger("A_Memorix.SparseBM25")
|
||||
|
||||
try:
|
||||
import jieba # type: ignore
|
||||
|
||||
HAS_JIEBA = True
|
||||
except Exception:
|
||||
HAS_JIEBA = False
|
||||
jieba = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseBM25Config:
|
||||
"""BM25 稀疏检索配置。"""
|
||||
|
||||
enabled: bool = True
|
||||
backend: str = "fts5"
|
||||
lazy_load: bool = True
|
||||
mode: str = "auto" # auto | fallback_only | hybrid
|
||||
tokenizer_mode: str = "jieba" # jieba | mixed | char_2gram
|
||||
jieba_user_dict: str = ""
|
||||
char_ngram_n: int = 2
|
||||
candidate_k: int = 80
|
||||
max_doc_len: int = 2000
|
||||
enable_ngram_fallback_index: bool = True
|
||||
enable_like_fallback: bool = False
|
||||
enable_relation_sparse_fallback: bool = True
|
||||
relation_candidate_k: int = 60
|
||||
relation_max_doc_len: int = 512
|
||||
unload_on_disable: bool = True
|
||||
shrink_memory_on_unload: bool = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.backend = str(self.backend or "fts5").strip().lower()
|
||||
self.mode = str(self.mode or "auto").strip().lower()
|
||||
self.tokenizer_mode = str(self.tokenizer_mode or "jieba").strip().lower()
|
||||
self.char_ngram_n = max(1, int(self.char_ngram_n))
|
||||
self.candidate_k = max(1, int(self.candidate_k))
|
||||
self.max_doc_len = max(0, int(self.max_doc_len))
|
||||
self.relation_candidate_k = max(1, int(self.relation_candidate_k))
|
||||
self.relation_max_doc_len = max(0, int(self.relation_max_doc_len))
|
||||
if self.backend != "fts5":
|
||||
raise ValueError(f"sparse.backend 暂仅支持 fts5: {self.backend}")
|
||||
if self.mode not in {"auto", "fallback_only", "hybrid"}:
|
||||
raise ValueError(f"sparse.mode 非法: {self.mode}")
|
||||
if self.tokenizer_mode not in {"jieba", "mixed", "char_2gram"}:
|
||||
raise ValueError(f"sparse.tokenizer_mode 非法: {self.tokenizer_mode}")
|
||||
|
||||
|
||||
class SparseBM25Index:
|
||||
"""
|
||||
基于 SQLite FTS5 的 BM25 检索适配层。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata_store: MetadataStore,
|
||||
config: Optional[SparseBM25Config] = None,
|
||||
):
|
||||
self.metadata_store = metadata_store
|
||||
self.config = config or SparseBM25Config()
|
||||
self._conn: Optional[sqlite3.Connection] = None
|
||||
self._loaded: bool = False
|
||||
self._jieba_dict_loaded: bool = False
|
||||
|
||||
@property
|
||||
def loaded(self) -> bool:
|
||||
return self._loaded and self._conn is not None
|
||||
|
||||
def ensure_loaded(self) -> bool:
|
||||
"""按需加载 FTS 连接与索引。"""
|
||||
if not self.config.enabled:
|
||||
return False
|
||||
if self.loaded:
|
||||
return True
|
||||
|
||||
db_path = self.metadata_store.get_db_path()
|
||||
conn = sqlite3.connect(
|
||||
str(db_path),
|
||||
check_same_thread=False,
|
||||
timeout=30.0,
|
||||
)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
conn.execute("PRAGMA temp_store=MEMORY")
|
||||
|
||||
if not self.metadata_store.ensure_fts_schema(conn=conn):
|
||||
conn.close()
|
||||
return False
|
||||
self.metadata_store.ensure_fts_backfilled(conn=conn)
|
||||
# 关系稀疏检索按独立开关加载，避免不必要的初始化开销。
|
||||
if self.config.enable_relation_sparse_fallback:
|
||||
self.metadata_store.ensure_relations_fts_schema(conn=conn)
|
||||
self.metadata_store.ensure_relations_fts_backfilled(conn=conn)
|
||||
if self.config.enable_ngram_fallback_index:
|
||||
self.metadata_store.ensure_paragraph_ngram_schema(conn=conn)
|
||||
self.metadata_store.ensure_paragraph_ngram_backfilled(
|
||||
n=self.config.char_ngram_n,
|
||||
conn=conn,
|
||||
)
|
||||
|
||||
self._conn = conn
|
||||
self._loaded = True
|
||||
self._prepare_tokenizer()
|
||||
logger.info(
|
||||
"SparseBM25Index loaded: "
|
||||
f"backend=fts5, tokenizer={self.config.tokenizer_mode}, mode={self.config.mode}"
|
||||
)
|
||||
return True
|
||||
|
||||
def _prepare_tokenizer(self) -> None:
|
||||
if self._jieba_dict_loaded:
|
||||
return
|
||||
if self.config.tokenizer_mode not in {"jieba", "mixed"}:
|
||||
return
|
||||
if not HAS_JIEBA:
|
||||
logger.warning("jieba 不可用，tokenizer 将退化为 char n-gram")
|
||||
return
|
||||
user_dict = str(self.config.jieba_user_dict or "").strip()
|
||||
if user_dict:
|
||||
try:
|
||||
jieba.load_userdict(user_dict) # type: ignore[union-attr]
|
||||
logger.info(f"已加载 jieba 用户词典: {user_dict}")
|
||||
except Exception as e:
|
||||
logger.warning(f"加载 jieba 用户词典失败: {e}")
|
||||
self._jieba_dict_loaded = True
|
||||
|
||||
def _tokenize_jieba(self, text: str) -> List[str]:
|
||||
if not HAS_JIEBA:
|
||||
return []
|
||||
try:
|
||||
tokens = list(jieba.cut_for_search(text)) # type: ignore[union-attr]
|
||||
return [t.strip().lower() for t in tokens if t and t.strip()]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def _tokenize_char_ngram(self, text: str, n: int) -> List[str]:
|
||||
compact = re.sub(r"\s+", "", text.lower())
|
||||
if not compact:
|
||||
return []
|
||||
if len(compact) < n:
|
||||
return [compact]
|
||||
return [compact[i : i + n] for i in range(0, len(compact) - n + 1)]
|
||||
|
||||
def _tokenize(self, text: str) -> List[str]:
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
|
||||
mode = self.config.tokenizer_mode
|
||||
if mode == "jieba":
|
||||
tokens = self._tokenize_jieba(text)
|
||||
if tokens:
|
||||
return list(dict.fromkeys(tokens))
|
||||
return self._tokenize_char_ngram(text, self.config.char_ngram_n)
|
||||
|
||||
if mode == "mixed":
|
||||
toks = self._tokenize_jieba(text)
|
||||
toks.extend(self._tokenize_char_ngram(text, self.config.char_ngram_n))
|
||||
return list(dict.fromkeys([t for t in toks if t]))
|
||||
|
||||
return list(dict.fromkeys(self._tokenize_char_ngram(text, self.config.char_ngram_n)))
|
||||
|
||||
def _build_match_query(self, tokens: List[str]) -> str:
|
||||
safe_tokens: List[str] = []
|
||||
for token in tokens:
|
||||
t = token.replace('"', '""').strip()
|
||||
if not t:
|
||||
continue
|
||||
safe_tokens.append(f'"{t}"')
|
||||
if not safe_tokens:
|
||||
return ""
|
||||
# 采用 OR 提升召回，再交由 RRF 和阈值做稳健排序。
|
||||
return " OR ".join(safe_tokens[:64])
|
||||
|
||||
def _fallback_substring_search(
|
||||
self,
|
||||
tokens: List[str],
|
||||
limit: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
当 FTS5 因分词不一致召回为空时，退化为子串匹配召回。
|
||||
|
||||
说明：
|
||||
- FTS 索引当前采用 unicode61 tokenizer。
|
||||
- 若查询 token 来源为 char n-gram 或中文词元，可能与索引 token 不一致。
|
||||
- 这里使用 SQL LIKE 做兜底，按命中 token 覆盖度打分。
|
||||
"""
|
||||
if not tokens:
|
||||
return []
|
||||
|
||||
# 去重并裁剪 token 数量，避免生成超长 SQL。
|
||||
uniq_tokens = [t for t in dict.fromkeys(tokens) if t]
|
||||
uniq_tokens = uniq_tokens[:32]
|
||||
if not uniq_tokens:
|
||||
return []
|
||||
|
||||
if self.config.enable_ngram_fallback_index:
|
||||
try:
|
||||
# 允许运行时切换开关后按需补齐 schema/回填。
|
||||
self.metadata_store.ensure_paragraph_ngram_schema(conn=self._conn)
|
||||
self.metadata_store.ensure_paragraph_ngram_backfilled(
|
||||
n=self.config.char_ngram_n,
|
||||
conn=self._conn,
|
||||
)
|
||||
rows = self.metadata_store.ngram_search_paragraphs(
|
||||
tokens=uniq_tokens,
|
||||
limit=limit,
|
||||
max_doc_len=self.config.max_doc_len,
|
||||
conn=self._conn,
|
||||
)
|
||||
if rows:
|
||||
return rows
|
||||
except Exception as e:
|
||||
logger.warning(f"ngram 倒排回退失败，将按配置决定是否使用 LIKE 回退: {e}")
|
||||
|
||||
if not self.config.enable_like_fallback:
|
||||
return []
|
||||
|
||||
conditions = " OR ".join(["p.content LIKE ?"] * len(uniq_tokens))
|
||||
params: List[Any] = [f"%{tok}%" for tok in uniq_tokens]
|
||||
scan_limit = max(int(limit) * 8, 200)
|
||||
params.append(scan_limit)
|
||||
|
||||
sql = f"""
|
||||
SELECT p.hash, p.content
|
||||
FROM paragraphs p
|
||||
WHERE (p.is_deleted IS NULL OR p.is_deleted = 0)
|
||||
AND ({conditions})
|
||||
LIMIT ?
|
||||
"""
|
||||
rows = self.metadata_store.query(sql, tuple(params))
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
scored: List[Dict[str, Any]] = []
|
||||
token_count = max(1, len(uniq_tokens))
|
||||
for row in rows:
|
||||
content = str(row.get("content") or "")
|
||||
content_low = content.lower()
|
||||
matched = [tok for tok in uniq_tokens if tok in content_low]
|
||||
if not matched:
|
||||
continue
|
||||
coverage = len(matched) / token_count
|
||||
length_bonus = sum(len(tok) for tok in matched) / max(1, len(content_low))
|
||||
# 兜底路径使用相对分，保持与上层接口兼容。
|
||||
fallback_score = coverage * 0.8 + length_bonus * 0.2
|
||||
scored.append(
|
||||
{
|
||||
"hash": row["hash"],
|
||||
"content": content[: self.config.max_doc_len] if self.config.max_doc_len > 0 else content,
|
||||
"bm25_score": -float(fallback_score),
|
||||
"fallback_score": float(fallback_score),
|
||||
}
|
||||
)
|
||||
|
||||
scored.sort(key=lambda x: x["fallback_score"], reverse=True)
|
||||
return scored[:limit]
|
||||
|
||||
def search(self, query: str, k: int = 20) -> List[Dict[str, Any]]:
|
||||
"""执行 BM25 检索。"""
|
||||
if not self.config.enabled:
|
||||
return []
|
||||
if self.config.lazy_load and not self.loaded:
|
||||
if not self.ensure_loaded():
|
||||
return []
|
||||
if not self.loaded:
|
||||
return []
|
||||
# 关系稀疏检索可独立开关，运行时开启后也能按需补齐 schema/回填。
|
||||
self.metadata_store.ensure_relations_fts_schema(conn=self._conn)
|
||||
self.metadata_store.ensure_relations_fts_backfilled(conn=self._conn)
|
||||
|
||||
tokens = self._tokenize(query)
|
||||
match_query = self._build_match_query(tokens)
|
||||
if not match_query:
|
||||
return []
|
||||
|
||||
limit = max(1, int(k))
|
||||
rows = self.metadata_store.fts_search_bm25(
|
||||
match_query=match_query,
|
||||
limit=limit,
|
||||
max_doc_len=self.config.max_doc_len,
|
||||
conn=self._conn,
|
||||
)
|
||||
if not rows:
|
||||
rows = self._fallback_substring_search(tokens=tokens, limit=limit)
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
for rank, row in enumerate(rows, start=1):
|
||||
bm25_score = float(row.get("bm25_score", 0.0))
|
||||
results.append(
|
||||
{
|
||||
"hash": row["hash"],
|
||||
"content": row["content"],
|
||||
"rank": rank,
|
||||
"bm25_score": bm25_score,
|
||||
"score": -bm25_score, # bm25 越小越相关，这里取反作为相对分数
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
def search_relations(self, query: str, k: int = 20) -> List[Dict[str, Any]]:
|
||||
"""执行关系稀疏检索（FTS5 + BM25）。"""
|
||||
if not self.config.enabled or not self.config.enable_relation_sparse_fallback:
|
||||
return []
|
||||
if self.config.lazy_load and not self.loaded:
|
||||
if not self.ensure_loaded():
|
||||
return []
|
||||
if not self.loaded:
|
||||
return []
|
||||
|
||||
tokens = self._tokenize(query)
|
||||
match_query = self._build_match_query(tokens)
|
||||
if not match_query:
|
||||
return []
|
||||
|
||||
rows = self.metadata_store.fts_search_relations_bm25(
|
||||
match_query=match_query,
|
||||
limit=max(1, int(k)),
|
||||
max_doc_len=self.config.relation_max_doc_len,
|
||||
conn=self._conn,
|
||||
)
|
||||
out: List[Dict[str, Any]] = []
|
||||
for rank, row in enumerate(rows, start=1):
|
||||
bm25_score = float(row.get("bm25_score", 0.0))
|
||||
out.append(
|
||||
{
|
||||
"hash": row["hash"],
|
||||
"subject": row["subject"],
|
||||
"predicate": row["predicate"],
|
||||
"object": row["object"],
|
||||
"content": row["content"],
|
||||
"rank": rank,
|
||||
"bm25_score": bm25_score,
|
||||
"score": -bm25_score,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
def upsert_paragraph(self, paragraph_hash: str) -> bool:
|
||||
if not self.loaded:
|
||||
return False
|
||||
return self.metadata_store.fts_upsert_paragraph(paragraph_hash, conn=self._conn)
|
||||
|
||||
def delete_paragraph(self, paragraph_hash: str) -> bool:
|
||||
if not self.loaded:
|
||||
return False
|
||||
return self.metadata_store.fts_delete_paragraph(paragraph_hash, conn=self._conn)
|
||||
|
||||
def unload(self) -> None:
|
||||
"""卸载 BM25 连接并尽量释放内存。"""
|
||||
if self._conn is not None:
|
||||
try:
|
||||
if self.config.shrink_memory_on_unload:
|
||||
self.metadata_store.shrink_memory(conn=self._conn)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._conn = None
|
||||
self._loaded = False
|
||||
logger.info("SparseBM25Index unloaded")
|
||||
|
||||
def stats(self) -> Dict[str, Any]:
|
||||
doc_count = 0
|
||||
if self.loaded:
|
||||
doc_count = self.metadata_store.fts_doc_count(conn=self._conn)
|
||||
return {
|
||||
"enabled": self.config.enabled,
|
||||
"backend": self.config.backend,
|
||||
"mode": self.config.mode,
|
||||
"tokenizer_mode": self.config.tokenizer_mode,
|
||||
"enable_ngram_fallback_index": self.config.enable_ngram_fallback_index,
|
||||
"enable_like_fallback": self.config.enable_like_fallback,
|
||||
"enable_relation_sparse_fallback": self.config.enable_relation_sparse_fallback,
|
||||
"loaded": self.loaded,
|
||||
"has_jieba": HAS_JIEBA,
|
||||
"doc_count": doc_count,
|
||||
}
|
||||
450
src/A_memorix/core/retrieval/threshold.py
Normal file
450
src/A_memorix/core/retrieval/threshold.py
Normal file
@@ -0,0 +1,450 @@
|
||||
"""
|
||||
动态阈值过滤器
|
||||
|
||||
根据检索结果的分布特征自适应调整过滤阈值。
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
from typing import List, Dict, Any, Optional, Tuple, Union
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
from src.common.logger import get_logger
|
||||
from .dual_path import RetrievalResult
|
||||
|
||||
logger = get_logger("A_Memorix.DynamicThresholdFilter")
|
||||
|
||||
|
||||
class ThresholdMethod(Enum):
|
||||
"""阈值计算方法"""
|
||||
|
||||
PERCENTILE = "percentile" # 百分位数
|
||||
STD_DEV = "std_dev" # 标准差
|
||||
GAP_DETECTION = "gap_detection" # 跳变检测
|
||||
ADAPTIVE = "adaptive" # 自适应（综合多种方法）
|
||||
|
||||
|
||||
@dataclass
|
||||
class ThresholdConfig:
|
||||
"""
|
||||
阈值配置
|
||||
|
||||
属性：
|
||||
method: 阈值计算方法
|
||||
min_threshold: 最小阈值（绝对值）
|
||||
max_threshold: 最大阈值（绝对值）
|
||||
percentile: 百分位数（用于percentile方法）
|
||||
std_multiplier: 标准差倍数（用于std_dev方法）
|
||||
min_results: 最少保留结果数
|
||||
enable_auto_adjust: 是否自动调整参数
|
||||
"""
|
||||
|
||||
method: ThresholdMethod = ThresholdMethod.ADAPTIVE
|
||||
min_threshold: float = 0.3
|
||||
max_threshold: float = 0.95
|
||||
percentile: float = 75.0 # 百分位数
|
||||
std_multiplier: float = 1.5 # 标准差倍数
|
||||
min_results: int = 3 # 最少保留结果数
|
||||
enable_auto_adjust: bool = True
|
||||
|
||||
def __post_init__(self):
|
||||
"""验证配置"""
|
||||
if not 0 <= self.min_threshold <= 1:
|
||||
raise ValueError(f"min_threshold必须在[0, 1]之间: {self.min_threshold}")
|
||||
|
||||
if not 0 <= self.max_threshold <= 1:
|
||||
raise ValueError(f"max_threshold必须在[0, 1]之间: {self.max_threshold}")
|
||||
|
||||
if self.min_threshold >= self.max_threshold:
|
||||
raise ValueError(f"min_threshold必须小于max_threshold")
|
||||
|
||||
if not 0 <= self.percentile <= 100:
|
||||
raise ValueError(f"percentile必须在[0, 100]之间: {self.percentile}")
|
||||
|
||||
if self.std_multiplier <= 0:
|
||||
raise ValueError(f"std_multiplier必须大于0: {self.std_multiplier}")
|
||||
|
||||
if self.min_results < 0:
|
||||
raise ValueError(f"min_results必须大于等于0: {self.min_results}")
|
||||
|
||||
|
||||
class DynamicThresholdFilter:
|
||||
"""
|
||||
动态阈值过滤器
|
||||
|
||||
功能：
|
||||
- 基于结果分布自适应计算阈值
|
||||
- 多种阈值计算方法
|
||||
- 自动参数调整
|
||||
- 统计信息收集
|
||||
|
||||
参数：
|
||||
config: 阈值配置
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Optional[ThresholdConfig] = None,
|
||||
):
|
||||
"""
|
||||
初始化动态阈值过滤器
|
||||
|
||||
Args:
|
||||
config: 阈值配置
|
||||
"""
|
||||
self.config = config or ThresholdConfig()
|
||||
|
||||
# 统计信息
|
||||
self._total_filtered = 0
|
||||
self._total_processed = 0
|
||||
self._threshold_history: List[float] = []
|
||||
|
||||
logger.info(
|
||||
f"DynamicThresholdFilter 初始化: "
|
||||
f"method={self.config.method.value}, "
|
||||
f"min_threshold={self.config.min_threshold}"
|
||||
)
|
||||
|
||||
def filter(
|
||||
self,
|
||||
results: List[RetrievalResult],
|
||||
return_threshold: bool = False,
|
||||
) -> Union[List[RetrievalResult], Tuple[List[RetrievalResult], float]]:
|
||||
"""
|
||||
过滤检索结果
|
||||
|
||||
Args:
|
||||
results: 检索结果列表
|
||||
return_threshold: 是否返回使用的阈值
|
||||
|
||||
Returns:
|
||||
过滤后的结果列表，或 (结果列表, 阈值) 元组
|
||||
"""
|
||||
if not results:
|
||||
logger.debug("结果列表为空，无需过滤")
|
||||
return ([], 0.0) if return_threshold else []
|
||||
|
||||
self._total_processed += len(results)
|
||||
|
||||
# 提取分数
|
||||
scores = np.array([r.score for r in results])
|
||||
|
||||
# 计算阈值
|
||||
threshold = self._compute_threshold(scores, results)
|
||||
|
||||
# 记录阈值
|
||||
self._threshold_history.append(threshold)
|
||||
|
||||
# 应用阈值过滤
|
||||
filtered_results = [
|
||||
r for r in results
|
||||
if r.score >= threshold
|
||||
]
|
||||
|
||||
# 确保至少保留min_results个结果
|
||||
if len(filtered_results) < self.config.min_results:
|
||||
# 按分数排序，取前min_results个
|
||||
sorted_results = sorted(results, key=lambda x: x.score, reverse=True)
|
||||
filtered_results = sorted_results[:self.config.min_results]
|
||||
threshold = filtered_results[-1].score if filtered_results else 0.0
|
||||
|
||||
self._total_filtered += len(results) - len(filtered_results)
|
||||
|
||||
logger.info(
|
||||
f"过滤完成: {len(results)} -> {len(filtered_results)} "
|
||||
f"(threshold={threshold:.3f})"
|
||||
)
|
||||
|
||||
if return_threshold:
|
||||
return filtered_results, threshold
|
||||
return filtered_results
|
||||
|
||||
def _compute_threshold(
|
||||
self,
|
||||
scores: np.ndarray,
|
||||
results: List[RetrievalResult],
|
||||
) -> float:
|
||||
"""
|
||||
计算阈值
|
||||
|
||||
Args:
|
||||
scores: 分数数组
|
||||
results: 检索结果列表
|
||||
|
||||
Returns:
|
||||
阈值
|
||||
"""
|
||||
if self.config.method == ThresholdMethod.PERCENTILE:
|
||||
threshold = self._percentile_threshold(scores)
|
||||
elif self.config.method == ThresholdMethod.STD_DEV:
|
||||
threshold = self._std_dev_threshold(scores)
|
||||
elif self.config.method == ThresholdMethod.GAP_DETECTION:
|
||||
threshold = self._gap_detection_threshold(scores)
|
||||
else: # ADAPTIVE
|
||||
# 自适应方法：综合多种方法
|
||||
thresholds = [
|
||||
self._percentile_threshold(scores),
|
||||
self._std_dev_threshold(scores),
|
||||
self._gap_detection_threshold(scores),
|
||||
]
|
||||
# 使用中位数作为最终阈值
|
||||
threshold = float(np.median(thresholds))
|
||||
|
||||
# 限制在[min_threshold, max_threshold]范围内
|
||||
threshold = np.clip(
|
||||
threshold,
|
||||
self.config.min_threshold,
|
||||
self.config.max_threshold,
|
||||
)
|
||||
|
||||
# 自动调整
|
||||
if self.config.enable_auto_adjust:
|
||||
threshold = self._auto_adjust_threshold(threshold, scores)
|
||||
|
||||
return float(threshold)
|
||||
|
||||
def _percentile_threshold(self, scores: np.ndarray) -> float:
|
||||
"""
|
||||
基于百分位数计算阈值
|
||||
|
||||
Args:
|
||||
scores: 分数数组
|
||||
|
||||
Returns:
|
||||
阈值
|
||||
"""
|
||||
percentile = self.config.percentile
|
||||
threshold = float(np.percentile(scores, percentile))
|
||||
|
||||
logger.debug(f"百分位数阈值: {threshold:.3f} (percentile={percentile})")
|
||||
return threshold
|
||||
|
||||
def _std_dev_threshold(self, scores: np.ndarray) -> float:
|
||||
"""
|
||||
基于标准差计算阈值
|
||||
|
||||
threshold = mean - std_multiplier * std
|
||||
|
||||
Args:
|
||||
scores: 分数数组
|
||||
|
||||
Returns:
|
||||
阈值
|
||||
"""
|
||||
mean = float(np.mean(scores))
|
||||
std = float(np.std(scores))
|
||||
multiplier = self.config.std_multiplier
|
||||
|
||||
threshold = mean - multiplier * std
|
||||
|
||||
logger.debug(f"标准差阈值: {threshold:.3f} (mean={mean:.3f}, std={std:.3f})")
|
||||
return threshold
|
||||
|
||||
def _gap_detection_threshold(self, scores: np.ndarray) -> float:
|
||||
"""
|
||||
基于跳变检测计算阈值
|
||||
|
||||
找到分数分布中最大的"跳变"位置，以此为阈值
|
||||
|
||||
Args:
|
||||
scores: 分数数组（降序排列）
|
||||
|
||||
Returns:
|
||||
阈值
|
||||
"""
|
||||
# 降序排列
|
||||
sorted_scores = np.sort(scores)[::-1]
|
||||
|
||||
if len(sorted_scores) < 2:
|
||||
return float(sorted_scores[0]) if len(sorted_scores) > 0 else 0.0
|
||||
|
||||
# 计算相邻分数的差值
|
||||
gaps = np.diff(sorted_scores)
|
||||
|
||||
# 找到最大的跳变位置
|
||||
max_gap_idx = int(np.argmax(gaps))
|
||||
|
||||
# 阈值为跳变后的分数
|
||||
threshold = float(sorted_scores[max_gap_idx + 1])
|
||||
|
||||
logger.debug(
|
||||
f"跳变检测阈值: {threshold:.3f} "
|
||||
f"(gap={gaps[max_gap_idx]:.3f}, idx={max_gap_idx})"
|
||||
)
|
||||
return threshold
|
||||
|
||||
def _auto_adjust_threshold(
|
||||
self,
|
||||
threshold: float,
|
||||
scores: np.ndarray,
|
||||
) -> float:
|
||||
"""
|
||||
自动调整阈值
|
||||
|
||||
基于历史阈值和当前分数分布调整
|
||||
|
||||
Args:
|
||||
threshold: 当前阈值
|
||||
scores: 分数数组
|
||||
|
||||
Returns:
|
||||
调整后的阈值
|
||||
"""
|
||||
if not self._threshold_history:
|
||||
return threshold
|
||||
|
||||
# 计算历史阈值的移动平均
|
||||
recent_thresholds = self._threshold_history[-10:] # 最近10次
|
||||
avg_threshold = float(np.mean(recent_thresholds))
|
||||
|
||||
# 当前阈值与历史平均的差异
|
||||
diff = threshold - avg_threshold
|
||||
|
||||
# 如果差异过大（>0.2），向历史平均靠拢
|
||||
if abs(diff) > 0.2:
|
||||
adjusted_threshold = avg_threshold + diff * 0.5 # 向中间靠拢50%
|
||||
logger.debug(
|
||||
f"阈值调整: {threshold:.3f} -> {adjusted_threshold:.3f} "
|
||||
f"(历史平均={avg_threshold:.3f})"
|
||||
)
|
||||
return adjusted_threshold
|
||||
|
||||
return threshold
|
||||
|
||||
def filter_by_confidence(
|
||||
self,
|
||||
results: List[RetrievalResult],
|
||||
min_confidence: float = 0.5,
|
||||
) -> List[RetrievalResult]:
|
||||
"""
|
||||
基于置信度过滤结果
|
||||
|
||||
Args:
|
||||
results: 检索结果列表
|
||||
min_confidence: 最小置信度
|
||||
|
||||
Returns:
|
||||
过滤后的结果列表
|
||||
"""
|
||||
filtered = []
|
||||
for result in results:
|
||||
# 对于关系结果，使用confidence字段
|
||||
if result.result_type == "relation":
|
||||
confidence = result.metadata.get("confidence", 1.0)
|
||||
if confidence >= min_confidence:
|
||||
filtered.append(result)
|
||||
else:
|
||||
# 对于段落结果，直接使用分数
|
||||
if result.score >= min_confidence:
|
||||
filtered.append(result)
|
||||
|
||||
logger.info(
|
||||
f"置信度过滤: {len(results)} -> {len(filtered)} "
|
||||
f"(min_confidence={min_confidence})"
|
||||
)
|
||||
|
||||
return filtered
|
||||
|
||||
def filter_by_diversity(
|
||||
self,
|
||||
results: List[RetrievalResult],
|
||||
similarity_threshold: float = 0.9,
|
||||
top_k: int = 10,
|
||||
) -> List[RetrievalResult]:
|
||||
"""
|
||||
基于多样性过滤结果（去除重复）
|
||||
|
||||
Args:
|
||||
results: 检索结果列表
|
||||
similarity_threshold: 相似度阈值（高于此值视为重复）
|
||||
top_k: 最多保留结果数
|
||||
|
||||
Returns:
|
||||
过滤后的结果列表
|
||||
"""
|
||||
if not results:
|
||||
return []
|
||||
|
||||
# 按分数排序
|
||||
sorted_results = sorted(results, key=lambda x: x.score, reverse=True)
|
||||
|
||||
# 贪心选择：选择与已选结果相似度低的结果
|
||||
selected = []
|
||||
selected_hashes = []
|
||||
|
||||
for result in sorted_results:
|
||||
if len(selected) >= top_k:
|
||||
break
|
||||
|
||||
# 检查与已选结果的相似度
|
||||
is_duplicate = False
|
||||
for selected_hash in selected_hashes:
|
||||
# 简单判断：基于hash的前缀
|
||||
if result.hash_value[:8] == selected_hash[:8]:
|
||||
is_duplicate = True
|
||||
break
|
||||
|
||||
if not is_duplicate:
|
||||
selected.append(result)
|
||||
selected_hashes.append(result.hash_value)
|
||||
|
||||
logger.info(
|
||||
f"多样性过滤: {len(results)} -> {len(selected)} "
|
||||
f"(similarity_threshold={similarity_threshold})"
|
||||
)
|
||||
|
||||
return selected
|
||||
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取统计信息
|
||||
|
||||
Returns:
|
||||
统计信息字典
|
||||
"""
|
||||
filter_rate = (
|
||||
self._total_filtered / self._total_processed
|
||||
if self._total_processed > 0
|
||||
else 0.0
|
||||
)
|
||||
|
||||
stats = {
|
||||
"config": {
|
||||
"method": self.config.method.value,
|
||||
"min_threshold": self.config.min_threshold,
|
||||
"max_threshold": self.config.max_threshold,
|
||||
"percentile": self.config.percentile,
|
||||
"std_multiplier": self.config.std_multiplier,
|
||||
"min_results": self.config.min_results,
|
||||
"enable_auto_adjust": self.config.enable_auto_adjust,
|
||||
},
|
||||
"statistics": {
|
||||
"total_processed": self._total_processed,
|
||||
"total_filtered": self._total_filtered,
|
||||
"filter_rate": filter_rate,
|
||||
"avg_threshold": float(np.mean(self._threshold_history))
|
||||
if self._threshold_history else 0.0,
|
||||
"threshold_count": len(self._threshold_history),
|
||||
},
|
||||
}
|
||||
|
||||
if self._threshold_history:
|
||||
stats["statistics"]["min_threshold_used"] = float(np.min(self._threshold_history))
|
||||
stats["statistics"]["max_threshold_used"] = float(np.max(self._threshold_history))
|
||||
|
||||
return stats
|
||||
|
||||
def reset_statistics(self) -> None:
|
||||
"""重置统计信息"""
|
||||
self._total_filtered = 0
|
||||
self._total_processed = 0
|
||||
self._threshold_history.clear()
|
||||
logger.info("统计信息已重置")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"DynamicThresholdFilter("
|
||||
f"method={self.config.method.value}, "
|
||||
f"min_threshold={self.config.min_threshold}, "
|
||||
f"filtered={self._total_filtered}/{self._total_processed})"
|
||||
)
|
||||
16
src/A_memorix/core/runtime/__init__.py
Normal file
16
src/A_memorix/core/runtime/__init__.py
Normal file
@@ -0,0 +1,16 @@
|
||||
"""SDK runtime exports for A_Memorix."""
|
||||
|
||||
from .search_runtime_initializer import (
|
||||
SearchRuntimeBundle,
|
||||
SearchRuntimeInitializer,
|
||||
build_search_runtime,
|
||||
)
|
||||
from .sdk_memory_kernel import KernelSearchRequest, SDKMemoryKernel
|
||||
|
||||
__all__ = [
|
||||
"SearchRuntimeBundle",
|
||||
"SearchRuntimeInitializer",
|
||||
"build_search_runtime",
|
||||
"KernelSearchRequest",
|
||||
"SDKMemoryKernel",
|
||||
]
|
||||
265
src/A_memorix/core/runtime/lifecycle_orchestrator.py
Normal file
265
src/A_memorix/core/runtime/lifecycle_orchestrator.py
Normal file
@@ -0,0 +1,265 @@
|
||||
"""Lifecycle bootstrap/teardown helpers extracted from plugin.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from src.common.logger import get_logger
|
||||
|
||||
from ...paths import default_data_dir, resolve_repo_path
|
||||
from ..embedding import create_embedding_api_adapter
|
||||
from ..retrieval import SparseBM25Config, SparseBM25Index
|
||||
from ..storage import (
|
||||
GraphStore,
|
||||
MetadataStore,
|
||||
QuantizationType,
|
||||
SparseMatrixFormat,
|
||||
VectorStore,
|
||||
)
|
||||
from ..utils.runtime_self_check import ensure_runtime_self_check
|
||||
from ..utils.relation_write_service import RelationWriteService
|
||||
|
||||
logger = get_logger("A_Memorix.LifecycleOrchestrator")
|
||||
|
||||
|
||||
async def ensure_initialized(plugin: Any) -> None:
|
||||
if plugin._initialized:
|
||||
plugin._runtime_ready = plugin._check_storage_ready()
|
||||
return
|
||||
|
||||
async with plugin._init_lock:
|
||||
if plugin._initialized:
|
||||
plugin._runtime_ready = plugin._check_storage_ready()
|
||||
return
|
||||
|
||||
logger.info("A_Memorix 插件正在异步初始化存储组件...")
|
||||
plugin._validate_runtime_config()
|
||||
await initialize_storage_async(plugin)
|
||||
report = await ensure_runtime_self_check(plugin, force=True)
|
||||
if not bool(report.get("ok", False)):
|
||||
logger.error(
|
||||
"A_Memorix runtime self-check failed: "
|
||||
f"{report.get('message', 'unknown')}; "
|
||||
"建议执行 python src/A_memorix/scripts/runtime_self_check.py --json"
|
||||
)
|
||||
|
||||
if plugin.graph_store and plugin.metadata_store:
|
||||
relation_count = plugin.metadata_store.count_relations()
|
||||
if relation_count > 0 and not plugin.graph_store.has_edge_hash_map():
|
||||
raise RuntimeError(
|
||||
"检测到 relations 数据存在但 edge-hash-map 为空。"
|
||||
" 请先执行 scripts/release_vnext_migrate.py migrate。"
|
||||
)
|
||||
|
||||
plugin._initialized = True
|
||||
plugin._runtime_ready = plugin._check_storage_ready()
|
||||
plugin._update_plugin_config()
|
||||
logger.info("A_Memorix 插件异步初始化成功")
|
||||
|
||||
|
||||
def start_background_tasks(plugin: Any) -> None:
|
||||
"""Start background tasks idempotently."""
|
||||
if not hasattr(plugin, "_episode_generation_task"):
|
||||
plugin._episode_generation_task = None
|
||||
|
||||
if (
|
||||
plugin.get_config("summarization.enabled", True)
|
||||
and plugin.get_config("schedule.enabled", True)
|
||||
and (plugin._scheduled_import_task is None or plugin._scheduled_import_task.done())
|
||||
):
|
||||
plugin._scheduled_import_task = asyncio.create_task(plugin._scheduled_import_loop())
|
||||
|
||||
if (
|
||||
plugin.get_config("advanced.enable_auto_save", True)
|
||||
and (plugin._auto_save_task is None or plugin._auto_save_task.done())
|
||||
):
|
||||
plugin._auto_save_task = asyncio.create_task(plugin._auto_save_loop())
|
||||
|
||||
if (
|
||||
plugin.get_config("person_profile.enabled", True)
|
||||
and (plugin._person_profile_refresh_task is None or plugin._person_profile_refresh_task.done())
|
||||
):
|
||||
plugin._person_profile_refresh_task = asyncio.create_task(plugin._person_profile_refresh_loop())
|
||||
|
||||
if plugin._memory_maintenance_task is None or plugin._memory_maintenance_task.done():
|
||||
plugin._memory_maintenance_task = asyncio.create_task(plugin._memory_maintenance_loop())
|
||||
|
||||
rv_cfg = plugin.get_config("retrieval.relation_vectorization", {}) or {}
|
||||
if isinstance(rv_cfg, dict):
|
||||
rv_enabled = bool(rv_cfg.get("enabled", False))
|
||||
rv_backfill = bool(rv_cfg.get("backfill_enabled", False))
|
||||
else:
|
||||
rv_enabled = False
|
||||
rv_backfill = False
|
||||
if rv_enabled and rv_backfill and (
|
||||
plugin._relation_vector_backfill_task is None or plugin._relation_vector_backfill_task.done()
|
||||
):
|
||||
plugin._relation_vector_backfill_task = asyncio.create_task(plugin._relation_vector_backfill_loop())
|
||||
|
||||
episode_task = getattr(plugin, "_episode_generation_task", None)
|
||||
episode_loop = getattr(plugin, "_episode_generation_loop", None)
|
||||
if (
|
||||
callable(episode_loop)
|
||||
and bool(plugin.get_config("episode.enabled", True))
|
||||
and bool(plugin.get_config("episode.generation_enabled", True))
|
||||
and (episode_task is None or episode_task.done())
|
||||
):
|
||||
plugin._episode_generation_task = asyncio.create_task(episode_loop())
|
||||
|
||||
|
||||
async def cancel_background_tasks(plugin: Any) -> None:
|
||||
"""Cancel all background tasks and wait for cleanup."""
|
||||
tasks = [
|
||||
("scheduled_import", plugin._scheduled_import_task),
|
||||
("auto_save", plugin._auto_save_task),
|
||||
("person_profile_refresh", plugin._person_profile_refresh_task),
|
||||
("memory_maintenance", plugin._memory_maintenance_task),
|
||||
("relation_vector_backfill", plugin._relation_vector_backfill_task),
|
||||
("episode_generation", getattr(plugin, "_episode_generation_task", None)),
|
||||
]
|
||||
for _, task in tasks:
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
|
||||
for name, task in tasks:
|
||||
if not task:
|
||||
continue
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.warning(f"后台任务 {name} 退出异常: {e}")
|
||||
|
||||
plugin._scheduled_import_task = None
|
||||
plugin._auto_save_task = None
|
||||
plugin._person_profile_refresh_task = None
|
||||
plugin._memory_maintenance_task = None
|
||||
plugin._relation_vector_backfill_task = None
|
||||
plugin._episode_generation_task = None
|
||||
|
||||
|
||||
async def initialize_storage_async(plugin: Any) -> None:
|
||||
"""Initialize storage components asynchronously."""
|
||||
data_dir_str = plugin.get_config("storage.data_dir", "./data")
|
||||
data_dir = resolve_repo_path(data_dir_str, fallback=default_data_dir())
|
||||
|
||||
logger.info(f"A_Memorix 数据存储路径: {data_dir}")
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
plugin.embedding_manager = create_embedding_api_adapter(
|
||||
batch_size=plugin.get_config("embedding.batch_size", 32),
|
||||
max_concurrent=plugin.get_config("embedding.max_concurrent", 5),
|
||||
default_dimension=plugin.get_config("embedding.dimension", 1024),
|
||||
model_name=plugin.get_config("embedding.model_name", "auto"),
|
||||
retry_config=plugin.get_config("embedding.retry", {}),
|
||||
)
|
||||
logger.info("嵌入 API 适配器初始化完成")
|
||||
|
||||
try:
|
||||
detected_dimension = await plugin.embedding_manager._detect_dimension()
|
||||
logger.info(f"嵌入维度检测成功: {detected_dimension}")
|
||||
except Exception as e:
|
||||
logger.warning(f"嵌入维度检测失败: {e}，使用默认值")
|
||||
detected_dimension = plugin.embedding_manager.default_dimension
|
||||
|
||||
quantization_str = plugin.get_config("embedding.quantization_type", "int8")
|
||||
if str(quantization_str or "").strip().lower() != "int8":
|
||||
raise ValueError("embedding.quantization_type 在 vNext 仅允许 int8(SQ8)。")
|
||||
quantization_type = QuantizationType.INT8
|
||||
|
||||
plugin.vector_store = VectorStore(
|
||||
dimension=detected_dimension,
|
||||
quantization_type=quantization_type,
|
||||
data_dir=data_dir / "vectors",
|
||||
)
|
||||
plugin.vector_store.min_train_threshold = plugin.get_config("embedding.min_train_threshold", 40)
|
||||
logger.info(
|
||||
"向量存储初始化完成（"
|
||||
f"维度: {detected_dimension}, "
|
||||
f"训练阈值: {plugin.vector_store.min_train_threshold}）"
|
||||
)
|
||||
|
||||
matrix_format_str = plugin.get_config("graph.sparse_matrix_format", "csr")
|
||||
matrix_format_map = {
|
||||
"csr": SparseMatrixFormat.CSR,
|
||||
"csc": SparseMatrixFormat.CSC,
|
||||
}
|
||||
matrix_format = matrix_format_map.get(matrix_format_str, SparseMatrixFormat.CSR)
|
||||
|
||||
plugin.graph_store = GraphStore(
|
||||
matrix_format=matrix_format,
|
||||
data_dir=data_dir / "graph",
|
||||
)
|
||||
logger.info("图存储初始化完成")
|
||||
|
||||
plugin.metadata_store = MetadataStore(data_dir=data_dir / "metadata")
|
||||
plugin.metadata_store.connect()
|
||||
logger.info("元数据存储初始化完成")
|
||||
|
||||
plugin.relation_write_service = RelationWriteService(
|
||||
metadata_store=plugin.metadata_store,
|
||||
graph_store=plugin.graph_store,
|
||||
vector_store=plugin.vector_store,
|
||||
embedding_manager=plugin.embedding_manager,
|
||||
)
|
||||
logger.info("关系写入服务初始化完成")
|
||||
|
||||
sparse_cfg_raw = plugin.get_config("retrieval.sparse", {}) or {}
|
||||
if not isinstance(sparse_cfg_raw, dict):
|
||||
sparse_cfg_raw = {}
|
||||
try:
|
||||
sparse_cfg = SparseBM25Config(**sparse_cfg_raw)
|
||||
except Exception as e:
|
||||
logger.warning(f"sparse 配置非法，回退默认配置: {e}")
|
||||
sparse_cfg = SparseBM25Config()
|
||||
plugin.sparse_index = SparseBM25Index(
|
||||
metadata_store=plugin.metadata_store,
|
||||
config=sparse_cfg,
|
||||
)
|
||||
logger.info(
|
||||
"稀疏检索组件初始化完成: "
|
||||
f"enabled={sparse_cfg.enabled}, "
|
||||
f"lazy_load={sparse_cfg.lazy_load}, "
|
||||
f"mode={sparse_cfg.mode}, "
|
||||
f"tokenizer={sparse_cfg.tokenizer_mode}"
|
||||
)
|
||||
if sparse_cfg.enabled and not sparse_cfg.lazy_load:
|
||||
plugin.sparse_index.ensure_loaded()
|
||||
|
||||
if plugin.vector_store.has_data():
|
||||
try:
|
||||
plugin.vector_store.load()
|
||||
logger.info(f"向量数据已加载，共 {plugin.vector_store.num_vectors} 个向量")
|
||||
except Exception as e:
|
||||
logger.warning(f"加载向量数据失败: {e}")
|
||||
|
||||
try:
|
||||
warmup_summary = plugin.vector_store.warmup_index(force_train=True)
|
||||
if warmup_summary.get("ok"):
|
||||
logger.info(
|
||||
"向量索引预热完成: "
|
||||
f"trained={warmup_summary.get('trained')}, "
|
||||
f"index_ntotal={warmup_summary.get('index_ntotal')}, "
|
||||
f"fallback_ntotal={warmup_summary.get('fallback_ntotal')}, "
|
||||
f"bin_count={warmup_summary.get('bin_count')}, "
|
||||
f"duration_ms={float(warmup_summary.get('duration_ms', 0.0)):.2f}"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"向量索引预热失败，继续启用 sparse 降级路径: "
|
||||
f"{warmup_summary.get('error', 'unknown')}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"向量索引预热异常，继续启用 sparse 降级路径: {e}")
|
||||
|
||||
if plugin.graph_store.has_data():
|
||||
try:
|
||||
plugin.graph_store.load()
|
||||
logger.info(f"图数据已加载，共 {plugin.graph_store.num_nodes} 个节点")
|
||||
except Exception as e:
|
||||
logger.warning(f"加载图数据失败: {e}")
|
||||
|
||||
logger.info(f"知识库数据目录: {data_dir}")
|
||||
4243
src/A_memorix/core/runtime/sdk_memory_kernel.py
Normal file
4243
src/A_memorix/core/runtime/sdk_memory_kernel.py
Normal file
File diff suppressed because it is too large
Load Diff
240
src/A_memorix/core/runtime/search_runtime_initializer.py
Normal file
240
src/A_memorix/core/runtime/search_runtime_initializer.py
Normal file
@@ -0,0 +1,240 @@
|
||||
"""Shared runtime initializer for Action/Tool/Command retrieval components."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from src.common.logger import get_logger
|
||||
|
||||
from ..retrieval import (
|
||||
DualPathRetriever,
|
||||
DualPathRetrieverConfig,
|
||||
DynamicThresholdFilter,
|
||||
FusionConfig,
|
||||
GraphRelationRecallConfig,
|
||||
RelationIntentConfig,
|
||||
RetrievalStrategy,
|
||||
SparseBM25Config,
|
||||
ThresholdConfig,
|
||||
ThresholdMethod,
|
||||
)
|
||||
|
||||
_logger = get_logger("A_Memorix.SearchRuntimeInitializer")
|
||||
|
||||
_REQUIRED_COMPONENT_KEYS = (
|
||||
"vector_store",
|
||||
"graph_store",
|
||||
"metadata_store",
|
||||
"embedding_manager",
|
||||
)
|
||||
|
||||
|
||||
def _get_config_value(config: Optional[dict], key: str, default: Any = None) -> Any:
|
||||
if not isinstance(config, dict):
|
||||
return default
|
||||
current: Any = config
|
||||
for part in key.split("."):
|
||||
if isinstance(current, dict) and part in current:
|
||||
current = current[part]
|
||||
else:
|
||||
return default
|
||||
return current
|
||||
|
||||
|
||||
def _safe_dict(value: Any) -> Dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _resolve_debug_enabled(plugin_config: Optional[dict]) -> bool:
|
||||
advanced = _get_config_value(plugin_config, "advanced", {})
|
||||
if isinstance(advanced, dict):
|
||||
return bool(advanced.get("debug", False))
|
||||
return bool(_get_config_value(plugin_config, "debug", False))
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchRuntimeBundle:
|
||||
"""Resolved runtime components and initialized retriever/filter."""
|
||||
|
||||
vector_store: Optional[Any] = None
|
||||
graph_store: Optional[Any] = None
|
||||
metadata_store: Optional[Any] = None
|
||||
embedding_manager: Optional[Any] = None
|
||||
sparse_index: Optional[Any] = None
|
||||
retriever: Optional[DualPathRetriever] = None
|
||||
threshold_filter: Optional[DynamicThresholdFilter] = None
|
||||
error: str = ""
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return (
|
||||
self.retriever is not None
|
||||
and self.vector_store is not None
|
||||
and self.graph_store is not None
|
||||
and self.metadata_store is not None
|
||||
and self.embedding_manager is not None
|
||||
)
|
||||
|
||||
|
||||
def _resolve_runtime_components(plugin_config: Optional[dict]) -> SearchRuntimeBundle:
|
||||
bundle = SearchRuntimeBundle(
|
||||
vector_store=_get_config_value(plugin_config, "vector_store"),
|
||||
graph_store=_get_config_value(plugin_config, "graph_store"),
|
||||
metadata_store=_get_config_value(plugin_config, "metadata_store"),
|
||||
embedding_manager=_get_config_value(plugin_config, "embedding_manager"),
|
||||
sparse_index=_get_config_value(plugin_config, "sparse_index"),
|
||||
)
|
||||
|
||||
missing_required = any(
|
||||
getattr(bundle, key) is None for key in _REQUIRED_COMPONENT_KEYS
|
||||
)
|
||||
if not missing_required:
|
||||
return bundle
|
||||
|
||||
try:
|
||||
from ...runtime_registry import get_runtime_components
|
||||
|
||||
instances = get_runtime_components()
|
||||
except Exception:
|
||||
instances = {}
|
||||
|
||||
if not isinstance(instances, dict) or not instances:
|
||||
return bundle
|
||||
|
||||
if bundle.vector_store is None:
|
||||
bundle.vector_store = instances.get("vector_store")
|
||||
if bundle.graph_store is None:
|
||||
bundle.graph_store = instances.get("graph_store")
|
||||
if bundle.metadata_store is None:
|
||||
bundle.metadata_store = instances.get("metadata_store")
|
||||
if bundle.embedding_manager is None:
|
||||
bundle.embedding_manager = instances.get("embedding_manager")
|
||||
if bundle.sparse_index is None:
|
||||
bundle.sparse_index = instances.get("sparse_index")
|
||||
return bundle
|
||||
|
||||
|
||||
def build_search_runtime(
|
||||
plugin_config: Optional[dict],
|
||||
logger_obj: Optional[Any],
|
||||
owner_tag: str,
|
||||
*,
|
||||
log_prefix: str = "",
|
||||
) -> SearchRuntimeBundle:
|
||||
"""Build retriever + threshold filter with unified fallback/config parsing."""
|
||||
|
||||
log = logger_obj or _logger
|
||||
owner = str(owner_tag or "runtime").strip().lower() or "runtime"
|
||||
prefix = str(log_prefix or "").strip()
|
||||
prefix_text = f"{prefix} " if prefix else ""
|
||||
|
||||
runtime = _resolve_runtime_components(plugin_config)
|
||||
if any(getattr(runtime, key) is None for key in _REQUIRED_COMPONENT_KEYS):
|
||||
runtime.error = "存储组件未完全初始化"
|
||||
log.warning(f"{prefix_text}[{owner}] 存储组件未完全初始化，无法使用检索功能")
|
||||
return runtime
|
||||
|
||||
sparse_cfg_raw = _safe_dict(_get_config_value(plugin_config, "retrieval.sparse", {}) or {})
|
||||
fusion_cfg_raw = _safe_dict(_get_config_value(plugin_config, "retrieval.fusion", {}) or {})
|
||||
relation_intent_cfg_raw = _safe_dict(
|
||||
_get_config_value(plugin_config, "retrieval.search.relation_intent", {}) or {}
|
||||
)
|
||||
graph_recall_cfg_raw = _safe_dict(
|
||||
_get_config_value(plugin_config, "retrieval.search.graph_recall", {}) or {}
|
||||
)
|
||||
|
||||
try:
|
||||
sparse_cfg = SparseBM25Config(**sparse_cfg_raw)
|
||||
except Exception as e:
|
||||
log.warning(f"{prefix_text}[{owner}] sparse 配置非法，回退默认: {e}")
|
||||
sparse_cfg = SparseBM25Config()
|
||||
|
||||
try:
|
||||
fusion_cfg = FusionConfig(**fusion_cfg_raw)
|
||||
except Exception as e:
|
||||
log.warning(f"{prefix_text}[{owner}] fusion 配置非法，回退默认: {e}")
|
||||
fusion_cfg = FusionConfig()
|
||||
|
||||
try:
|
||||
relation_intent_cfg = RelationIntentConfig(**relation_intent_cfg_raw)
|
||||
except Exception as e:
|
||||
log.warning(f"{prefix_text}[{owner}] relation_intent 配置非法，回退默认: {e}")
|
||||
relation_intent_cfg = RelationIntentConfig()
|
||||
|
||||
try:
|
||||
graph_recall_cfg = GraphRelationRecallConfig(**graph_recall_cfg_raw)
|
||||
except Exception as e:
|
||||
log.warning(f"{prefix_text}[{owner}] graph_recall 配置非法，回退默认: {e}")
|
||||
graph_recall_cfg = GraphRelationRecallConfig()
|
||||
|
||||
try:
|
||||
config = DualPathRetrieverConfig(
|
||||
top_k_paragraphs=_get_config_value(plugin_config, "retrieval.top_k_paragraphs", 20),
|
||||
top_k_relations=_get_config_value(plugin_config, "retrieval.top_k_relations", 10),
|
||||
top_k_final=_get_config_value(plugin_config, "retrieval.top_k_final", 10),
|
||||
alpha=_get_config_value(plugin_config, "retrieval.alpha", 0.5),
|
||||
enable_ppr=_get_config_value(plugin_config, "retrieval.enable_ppr", True),
|
||||
ppr_alpha=_get_config_value(plugin_config, "retrieval.ppr_alpha", 0.85),
|
||||
ppr_timeout_seconds=_get_config_value(
|
||||
plugin_config, "retrieval.ppr_timeout_seconds", 1.5
|
||||
),
|
||||
ppr_concurrency_limit=_get_config_value(
|
||||
plugin_config, "retrieval.ppr_concurrency_limit", 4
|
||||
),
|
||||
enable_parallel=_get_config_value(plugin_config, "retrieval.enable_parallel", True),
|
||||
retrieval_strategy=RetrievalStrategy.DUAL_PATH,
|
||||
debug=_resolve_debug_enabled(plugin_config),
|
||||
sparse=sparse_cfg,
|
||||
fusion=fusion_cfg,
|
||||
relation_intent=relation_intent_cfg,
|
||||
graph_recall=graph_recall_cfg,
|
||||
)
|
||||
|
||||
runtime.retriever = DualPathRetriever(
|
||||
vector_store=runtime.vector_store,
|
||||
graph_store=runtime.graph_store,
|
||||
metadata_store=runtime.metadata_store,
|
||||
embedding_manager=runtime.embedding_manager,
|
||||
sparse_index=runtime.sparse_index,
|
||||
config=config,
|
||||
)
|
||||
|
||||
threshold_config = ThresholdConfig(
|
||||
method=ThresholdMethod.ADAPTIVE,
|
||||
min_threshold=_get_config_value(plugin_config, "threshold.min_threshold", 0.3),
|
||||
max_threshold=_get_config_value(plugin_config, "threshold.max_threshold", 0.95),
|
||||
percentile=_get_config_value(plugin_config, "threshold.percentile", 75.0),
|
||||
std_multiplier=_get_config_value(plugin_config, "threshold.std_multiplier", 1.5),
|
||||
min_results=_get_config_value(plugin_config, "threshold.min_results", 3),
|
||||
enable_auto_adjust=_get_config_value(plugin_config, "threshold.enable_auto_adjust", True),
|
||||
)
|
||||
runtime.threshold_filter = DynamicThresholdFilter(threshold_config)
|
||||
runtime.error = ""
|
||||
log.info(f"{prefix_text}[{owner}] 检索运行时初始化完成")
|
||||
except Exception as e:
|
||||
runtime.retriever = None
|
||||
runtime.threshold_filter = None
|
||||
runtime.error = str(e)
|
||||
log.error(f"{prefix_text}[{owner}] 检索运行时初始化失败: {e}")
|
||||
|
||||
return runtime
|
||||
|
||||
|
||||
class SearchRuntimeInitializer:
|
||||
"""Compatibility wrapper around the function style initializer."""
|
||||
|
||||
@staticmethod
|
||||
def build_search_runtime(
|
||||
plugin_config: Optional[dict],
|
||||
logger_obj: Optional[Any],
|
||||
owner_tag: str,
|
||||
*,
|
||||
log_prefix: str = "",
|
||||
) -> SearchRuntimeBundle:
|
||||
return build_search_runtime(
|
||||
plugin_config=plugin_config,
|
||||
logger_obj=logger_obj,
|
||||
owner_tag=owner_tag,
|
||||
log_prefix=log_prefix,
|
||||
)
|
||||
53
src/A_memorix/core/storage/__init__.py
Normal file
53
src/A_memorix/core/storage/__init__.py
Normal file
@@ -0,0 +1,53 @@
|
||||
"""存储层"""
|
||||
|
||||
from .vector_store import VectorStore, QuantizationType
|
||||
from .graph_store import GraphStore, SparseMatrixFormat
|
||||
from .metadata_store import MetadataStore
|
||||
from .knowledge_types import (
|
||||
ImportStrategy,
|
||||
KnowledgeType,
|
||||
allowed_import_strategy_values,
|
||||
allowed_knowledge_type_values,
|
||||
get_knowledge_type_from_string,
|
||||
get_import_strategy_from_string,
|
||||
parse_import_strategy,
|
||||
resolve_stored_knowledge_type,
|
||||
should_extract_relations,
|
||||
get_default_chunk_size,
|
||||
get_type_display_name,
|
||||
validate_stored_knowledge_type,
|
||||
)
|
||||
from .type_detection import (
|
||||
detect_knowledge_type,
|
||||
get_type_from_user_input,
|
||||
looks_like_factual_text,
|
||||
looks_like_quote_text,
|
||||
looks_like_structured_text,
|
||||
select_import_strategy,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"VectorStore",
|
||||
"GraphStore",
|
||||
"MetadataStore",
|
||||
"QuantizationType",
|
||||
"SparseMatrixFormat",
|
||||
"ImportStrategy",
|
||||
"KnowledgeType",
|
||||
"allowed_import_strategy_values",
|
||||
"allowed_knowledge_type_values",
|
||||
"get_knowledge_type_from_string",
|
||||
"get_import_strategy_from_string",
|
||||
"parse_import_strategy",
|
||||
"resolve_stored_knowledge_type",
|
||||
"should_extract_relations",
|
||||
"get_default_chunk_size",
|
||||
"get_type_display_name",
|
||||
"validate_stored_knowledge_type",
|
||||
"detect_knowledge_type",
|
||||
"get_type_from_user_input",
|
||||
"looks_like_factual_text",
|
||||
"looks_like_quote_text",
|
||||
"looks_like_structured_text",
|
||||
"select_import_strategy",
|
||||
]
|
||||
1448
src/A_memorix/core/storage/graph_store.py
Normal file
1448
src/A_memorix/core/storage/graph_store.py
Normal file
File diff suppressed because it is too large
Load Diff
183
src/A_memorix/core/storage/knowledge_types.py
Normal file
183
src/A_memorix/core/storage/knowledge_types.py
Normal file
@@ -0,0 +1,183 @@
|
||||
"""Knowledge type and import strategy helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
class KnowledgeType(str, Enum):
|
||||
"""持久化到 paragraphs.knowledge_type 的合法类型。"""
|
||||
|
||||
STRUCTURED = "structured"
|
||||
NARRATIVE = "narrative"
|
||||
FACTUAL = "factual"
|
||||
QUOTE = "quote"
|
||||
MIXED = "mixed"
|
||||
|
||||
|
||||
class ImportStrategy(str, Enum):
|
||||
"""文本导入阶段的策略选择。"""
|
||||
|
||||
AUTO = "auto"
|
||||
NARRATIVE = "narrative"
|
||||
FACTUAL = "factual"
|
||||
QUOTE = "quote"
|
||||
|
||||
|
||||
def allowed_knowledge_type_values() -> tuple[str, ...]:
|
||||
return tuple(item.value for item in KnowledgeType)
|
||||
|
||||
|
||||
def allowed_import_strategy_values() -> tuple[str, ...]:
|
||||
return tuple(item.value for item in ImportStrategy)
|
||||
|
||||
|
||||
def get_knowledge_type_from_string(type_str: Any) -> Optional[KnowledgeType]:
|
||||
"""从字符串解析合法的落库知识类型。"""
|
||||
|
||||
if not isinstance(type_str, str):
|
||||
return None
|
||||
normalized = type_str.lower().strip()
|
||||
try:
|
||||
return KnowledgeType(normalized)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def get_import_strategy_from_string(value: Any) -> Optional[ImportStrategy]:
|
||||
"""从字符串解析文本导入策略。"""
|
||||
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
normalized = value.lower().strip()
|
||||
try:
|
||||
return ImportStrategy(normalized)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_import_strategy(value: Any, default: ImportStrategy = ImportStrategy.AUTO) -> ImportStrategy:
|
||||
"""解析 import strategy；非法值直接报错。"""
|
||||
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, ImportStrategy):
|
||||
return value
|
||||
|
||||
normalized = str(value or "").strip().lower()
|
||||
if not normalized:
|
||||
return default
|
||||
|
||||
strategy = get_import_strategy_from_string(normalized)
|
||||
if strategy is None:
|
||||
allowed = "/".join(allowed_import_strategy_values())
|
||||
raise ValueError(f"strategy_override 必须为 {allowed}")
|
||||
return strategy
|
||||
|
||||
|
||||
def validate_stored_knowledge_type(value: Any) -> KnowledgeType:
|
||||
"""校验写库 knowledge_type，仅允许合法落库类型。"""
|
||||
|
||||
if isinstance(value, KnowledgeType):
|
||||
return value
|
||||
|
||||
resolved = get_knowledge_type_from_string(value)
|
||||
if resolved is None:
|
||||
allowed = "/".join(allowed_knowledge_type_values())
|
||||
raise ValueError(f"knowledge_type 必须为 {allowed}")
|
||||
return resolved
|
||||
|
||||
|
||||
def resolve_stored_knowledge_type(
|
||||
value: Any,
|
||||
*,
|
||||
content: str = "",
|
||||
allow_legacy: bool = False,
|
||||
unknown_fallback: Optional[KnowledgeType] = None,
|
||||
) -> KnowledgeType:
|
||||
"""
|
||||
将策略/字符串/旧值解析为合法落库类型。
|
||||
|
||||
`allow_legacy=True` 仅供迁移使用。
|
||||
"""
|
||||
|
||||
if isinstance(value, KnowledgeType):
|
||||
return value
|
||||
|
||||
if isinstance(value, ImportStrategy):
|
||||
if value == ImportStrategy.AUTO:
|
||||
if not str(content or "").strip():
|
||||
raise ValueError("knowledge_type=auto 需要 content 才能推断")
|
||||
from .type_detection import detect_knowledge_type
|
||||
|
||||
return detect_knowledge_type(content)
|
||||
return KnowledgeType(value.value)
|
||||
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
if str(content or "").strip():
|
||||
from .type_detection import detect_knowledge_type
|
||||
|
||||
return detect_knowledge_type(content)
|
||||
raise ValueError("knowledge_type 不能为空")
|
||||
|
||||
direct = get_knowledge_type_from_string(raw)
|
||||
if direct is not None:
|
||||
return direct
|
||||
|
||||
strategy = get_import_strategy_from_string(raw)
|
||||
if strategy is not None:
|
||||
return resolve_stored_knowledge_type(strategy, content=content)
|
||||
|
||||
if allow_legacy:
|
||||
normalized = raw.lower()
|
||||
if normalized == "imported":
|
||||
return KnowledgeType.FACTUAL
|
||||
if str(content or "").strip():
|
||||
from .type_detection import detect_knowledge_type
|
||||
|
||||
detected = detect_knowledge_type(content)
|
||||
if detected is not None:
|
||||
return detected
|
||||
if unknown_fallback is not None:
|
||||
return unknown_fallback
|
||||
|
||||
allowed = "/".join(allowed_knowledge_type_values())
|
||||
raise ValueError(f"非法 knowledge_type: {raw}（仅允许 {allowed}）")
|
||||
|
||||
|
||||
def should_extract_relations(knowledge_type: KnowledgeType) -> bool:
|
||||
"""判断是否应该做关系抽取。"""
|
||||
|
||||
return knowledge_type in [
|
||||
KnowledgeType.STRUCTURED,
|
||||
KnowledgeType.FACTUAL,
|
||||
KnowledgeType.MIXED,
|
||||
]
|
||||
|
||||
|
||||
def get_default_chunk_size(knowledge_type: KnowledgeType) -> int:
|
||||
"""获取默认分块大小。"""
|
||||
|
||||
chunk_sizes = {
|
||||
KnowledgeType.STRUCTURED: 300,
|
||||
KnowledgeType.NARRATIVE: 800,
|
||||
KnowledgeType.FACTUAL: 500,
|
||||
KnowledgeType.QUOTE: 400,
|
||||
KnowledgeType.MIXED: 500,
|
||||
}
|
||||
return chunk_sizes.get(knowledge_type, 500)
|
||||
|
||||
|
||||
def get_type_display_name(knowledge_type: KnowledgeType) -> str:
|
||||
"""获取知识类型中文名称。"""
|
||||
|
||||
display_names = {
|
||||
KnowledgeType.STRUCTURED: "结构化知识",
|
||||
KnowledgeType.NARRATIVE: "叙事性文本",
|
||||
KnowledgeType.FACTUAL: "事实陈述",
|
||||
KnowledgeType.QUOTE: "引用文本",
|
||||
KnowledgeType.MIXED: "混合类型",
|
||||
}
|
||||
return display_names.get(knowledge_type, "未知类型")
|
||||
5959
src/A_memorix/core/storage/metadata_store.py
Normal file
5959
src/A_memorix/core/storage/metadata_store.py
Normal file
File diff suppressed because it is too large
Load Diff
137
src/A_memorix/core/storage/type_detection.py
Normal file
137
src/A_memorix/core/storage/type_detection.py
Normal file
@@ -0,0 +1,137 @@
|
||||
"""Heuristic detection for import strategies and stored knowledge types."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from .knowledge_types import (
|
||||
ImportStrategy,
|
||||
KnowledgeType,
|
||||
parse_import_strategy,
|
||||
resolve_stored_knowledge_type,
|
||||
)
|
||||
|
||||
|
||||
_NARRATIVE_MARKERS = [
|
||||
r"然后",
|
||||
r"接着",
|
||||
r"于是",
|
||||
r"后来",
|
||||
r"最后",
|
||||
r"突然",
|
||||
r"一天",
|
||||
r"曾经",
|
||||
r"有一次",
|
||||
r"从前",
|
||||
r"说道",
|
||||
r"问道",
|
||||
r"想着",
|
||||
r"觉得",
|
||||
]
|
||||
_FACTUAL_MARKERS = [
|
||||
r"是",
|
||||
r"有",
|
||||
r"在",
|
||||
r"为",
|
||||
r"属于",
|
||||
r"位于",
|
||||
r"包含",
|
||||
r"拥有",
|
||||
r"成立于",
|
||||
r"出生于",
|
||||
]
|
||||
|
||||
|
||||
def _non_empty_lines(content: str) -> list[str]:
|
||||
return [line for line in str(content or "").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def looks_like_structured_text(content: str) -> bool:
|
||||
text = str(content or "").strip()
|
||||
if "|" not in text or text.count("|") < 2:
|
||||
return False
|
||||
parts = text.split("|")
|
||||
return len(parts) == 3 and all(part.strip() for part in parts)
|
||||
|
||||
|
||||
def looks_like_quote_text(content: str) -> bool:
|
||||
lines = _non_empty_lines(content)
|
||||
if len(lines) < 5:
|
||||
return False
|
||||
avg_len = sum(len(line) for line in lines) / len(lines)
|
||||
return avg_len < 20
|
||||
|
||||
|
||||
def looks_like_narrative_text(content: str) -> bool:
|
||||
text = str(content or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
|
||||
narrative_score = sum(1 for marker in _NARRATIVE_MARKERS if re.search(marker, text))
|
||||
has_dialogue = bool(re.search(r'["「『].*?["」』]', text))
|
||||
has_chapter = any(token in text[:500] for token in ("Chapter", "CHAPTER", "###"))
|
||||
return has_chapter or has_dialogue or narrative_score >= 2
|
||||
|
||||
|
||||
def looks_like_factual_text(content: str) -> bool:
|
||||
text = str(content or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
if looks_like_structured_text(text) or looks_like_quote_text(text):
|
||||
return False
|
||||
|
||||
factual_score = sum(1 for marker in _FACTUAL_MARKERS if re.search(r"\s*" + marker + r"\s*", text))
|
||||
if factual_score <= 0:
|
||||
return False
|
||||
|
||||
if len(text) <= 240:
|
||||
return True
|
||||
return factual_score >= 2 and not looks_like_narrative_text(text)
|
||||
|
||||
|
||||
def select_import_strategy(
|
||||
content: str,
|
||||
*,
|
||||
override: Optional[str | ImportStrategy] = None,
|
||||
chat_log: bool = False,
|
||||
) -> ImportStrategy:
|
||||
"""文本导入策略选择：override > quote > factual > narrative。"""
|
||||
|
||||
if chat_log:
|
||||
return ImportStrategy.NARRATIVE
|
||||
|
||||
strategy = parse_import_strategy(override, default=ImportStrategy.AUTO)
|
||||
if strategy != ImportStrategy.AUTO:
|
||||
return strategy
|
||||
|
||||
if looks_like_quote_text(content):
|
||||
return ImportStrategy.QUOTE
|
||||
if looks_like_factual_text(content):
|
||||
return ImportStrategy.FACTUAL
|
||||
return ImportStrategy.NARRATIVE
|
||||
|
||||
|
||||
def detect_knowledge_type(content: str) -> KnowledgeType:
|
||||
"""自动检测落库 knowledge_type；无法可靠判断时回退 mixed。"""
|
||||
|
||||
text = str(content or "").strip()
|
||||
if not text:
|
||||
return KnowledgeType.MIXED
|
||||
if looks_like_structured_text(text):
|
||||
return KnowledgeType.STRUCTURED
|
||||
if looks_like_quote_text(text):
|
||||
return KnowledgeType.QUOTE
|
||||
if looks_like_factual_text(text):
|
||||
return KnowledgeType.FACTUAL
|
||||
if looks_like_narrative_text(text):
|
||||
return KnowledgeType.NARRATIVE
|
||||
return KnowledgeType.MIXED
|
||||
|
||||
|
||||
def get_type_from_user_input(type_hint: Optional[str], content: str) -> KnowledgeType:
|
||||
"""优先使用显式 type_hint，否则自动检测。"""
|
||||
|
||||
if type_hint:
|
||||
return resolve_stored_knowledge_type(type_hint, content=content)
|
||||
return detect_knowledge_type(content)
|
||||
776
src/A_memorix/core/storage/vector_store.py
Normal file
776
src/A_memorix/core/storage/vector_store.py
Normal file
@@ -0,0 +1,776 @@
|
||||
"""
|
||||
向量存储模块
|
||||
|
||||
基于Faiss的高效向量存储与检索，支持SQ8量化、Append-Only磁盘存储和内存映射。
|
||||
"""
|
||||
|
||||
import os
|
||||
import pickle
|
||||
import hashlib
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union, Tuple, List, Dict, Set, Any
|
||||
import random
|
||||
import threading # Added threading import
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import faiss
|
||||
HAS_FAISS = True
|
||||
except ImportError:
|
||||
HAS_FAISS = False
|
||||
|
||||
from src.common.logger import get_logger
|
||||
from ..utils.quantization import QuantizationType
|
||||
from ..utils.io import atomic_write, atomic_save_path
|
||||
|
||||
logger = get_logger("A_Memorix.VectorStore")
|
||||
|
||||
|
||||
class VectorStore:
|
||||
"""
|
||||
向量存储类 (SQ8 + Append-Only Disk)
|
||||
|
||||
特性：
|
||||
- 索引: IndexIDMap2(IndexScalarQuantizer(QT_8bit))
|
||||
- 存储: float16 on-disk binary (vectors.bin)
|
||||
- 内存: 仅索引常驻 RAM (<512MB for 100k vectors)
|
||||
- ID: SHA1-based stable int64 IDs
|
||||
- 一致性: 强制 L2 Normalization (IP == Cosine)
|
||||
"""
|
||||
|
||||
# 默认训练触发阈值 (40 样本，过大可能导致小数据集不生效，过小可能量化退化)
|
||||
DEFAULT_MIN_TRAIN = 40
|
||||
# 强制训练样本量
|
||||
TRAIN_SIZE = 10000
|
||||
# 储水池采样上限 (流式处理前 50k 数据)
|
||||
RESERVOIR_CAPACITY = 10000
|
||||
RESERVOIR_SAMPLE_SCOPE = 50000
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dimension: int,
|
||||
quantization_type: QuantizationType = QuantizationType.INT8,
|
||||
index_type: str = "sq8",
|
||||
data_dir: Optional[Union[str, Path]] = None,
|
||||
use_mmap: bool = True,
|
||||
buffer_size: int = 1024,
|
||||
):
|
||||
if not HAS_FAISS:
|
||||
raise ImportError("Faiss 未安装，请安装: pip install faiss-cpu")
|
||||
|
||||
self.dimension = dimension
|
||||
self.data_dir = Path(data_dir) if data_dir else None
|
||||
if self.data_dir:
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
if quantization_type != QuantizationType.INT8:
|
||||
raise ValueError(
|
||||
"vNext 仅支持 quantization_type=int8(SQ8)。"
|
||||
" 请更新配置并执行 scripts/release_vnext_migrate.py migrate。"
|
||||
)
|
||||
normalized_index_type = str(index_type or "sq8").strip().lower()
|
||||
if normalized_index_type not in {"sq8", "int8"}:
|
||||
raise ValueError(
|
||||
"vNext 仅支持 index_type=sq8。"
|
||||
" 请更新配置并执行 scripts/release_vnext_migrate.py migrate。"
|
||||
)
|
||||
self.quantization_type = QuantizationType.INT8
|
||||
self.index_type = "sq8"
|
||||
self.buffer_size = buffer_size
|
||||
|
||||
self._index: Optional[faiss.IndexIDMap2] = None
|
||||
self._init_index()
|
||||
|
||||
self._is_trained = False
|
||||
self._vector_norm = "l2"
|
||||
|
||||
# Fallback Index (Flat) - 用于在 SQ8 训练完成前提供检索能力
|
||||
# 必须使用 IndexIDMap2 以保证 ID 与主索引一致
|
||||
self._fallback_index: Optional[faiss.IndexIDMap2] = None
|
||||
self._init_fallback_index()
|
||||
|
||||
self._known_hashes: Set[str] = set()
|
||||
self._deleted_ids: Set[int] = set()
|
||||
|
||||
self._reservoir_buffer: List[np.ndarray] = []
|
||||
self._seen_count_for_reservoir = 0
|
||||
|
||||
self._write_buffer_vecs: List[np.ndarray] = []
|
||||
self._write_buffer_ids: List[int] = []
|
||||
|
||||
self._total_added = 0
|
||||
self._total_deleted = 0
|
||||
self._bin_count = 0
|
||||
|
||||
# Thread safety lock
|
||||
self._lock = threading.RLock()
|
||||
|
||||
logger.info(f"VectorStore Init: dim={dimension}, SQ8 Mode, Append-Only Storage")
|
||||
|
||||
def _init_index(self):
|
||||
"""初始化空的 Faiss 索引"""
|
||||
quantizer = faiss.IndexScalarQuantizer(
|
||||
self.dimension,
|
||||
faiss.ScalarQuantizer.QT_8bit,
|
||||
faiss.METRIC_INNER_PRODUCT
|
||||
)
|
||||
self._index = faiss.IndexIDMap2(quantizer)
|
||||
self._is_trained = False
|
||||
|
||||
def _init_fallback_index(self):
|
||||
"""初始化 Flat 回退索引"""
|
||||
flat_index = faiss.IndexFlatIP(self.dimension)
|
||||
self._fallback_index = faiss.IndexIDMap2(flat_index)
|
||||
logger.debug("Fallback index (Flat) initialized.")
|
||||
|
||||
@staticmethod
|
||||
def _generate_id(key: str) -> int:
|
||||
"""生成稳定的 int64 ID (SHA1 截断)"""
|
||||
h = hashlib.sha1(key.encode("utf-8")).digest()
|
||||
val = int.from_bytes(h[:8], byteorder="big", signed=False)
|
||||
return val & 0x7FFFFFFFFFFFFFFF
|
||||
|
||||
@property
|
||||
def _bin_path(self) -> Path:
|
||||
return self.data_dir / "vectors.bin"
|
||||
|
||||
@property
|
||||
def _ids_bin_path(self) -> Path:
|
||||
return self.data_dir / "vectors_ids.bin"
|
||||
|
||||
@property
|
||||
def _int_to_str_map(self) -> Dict[int, str]:
|
||||
"""Lazy build volatile map from known hashes"""
|
||||
# Note: This is read-heavy and cached, might need lock if _known_hashes updates concurrently
|
||||
# But add/delete are now locked, so checking len mismatch is somewhat safe-ish for quick dirty cache
|
||||
if not hasattr(self, "_cached_map") or len(self._cached_map) != len(self._known_hashes):
|
||||
with self._lock: # Protect cache rebuild
|
||||
self._cached_map = {self._generate_id(k): k for k in self._known_hashes}
|
||||
return self._cached_map
|
||||
|
||||
def add(self, vectors: np.ndarray, ids: List[str]) -> int:
|
||||
with self._lock:
|
||||
if vectors.shape[1] != self.dimension:
|
||||
raise ValueError(f"Dimension mismatch: {vectors.shape[1]} vs {self.dimension}")
|
||||
|
||||
vectors = np.ascontiguousarray(vectors, dtype=np.float32)
|
||||
faiss.normalize_L2(vectors)
|
||||
|
||||
processed_vecs = []
|
||||
processed_int_ids = []
|
||||
|
||||
for i, str_id in enumerate(ids):
|
||||
if str_id in self._known_hashes:
|
||||
continue
|
||||
|
||||
int_id = self._generate_id(str_id)
|
||||
self._known_hashes.add(str_id)
|
||||
|
||||
processed_vecs.append(vectors[i])
|
||||
processed_int_ids.append(int_id)
|
||||
|
||||
if not processed_vecs:
|
||||
return 0
|
||||
|
||||
batch_vecs = np.array(processed_vecs, dtype=np.float32)
|
||||
batch_ids = np.array(processed_int_ids, dtype=np.int64)
|
||||
|
||||
self._write_buffer_vecs.append(batch_vecs)
|
||||
self._write_buffer_ids.extend(processed_int_ids)
|
||||
|
||||
if len(self._write_buffer_ids) >= self.buffer_size:
|
||||
self._flush_write_buffer_unlocked()
|
||||
|
||||
if not self._is_trained:
|
||||
# 双写到回退索引
|
||||
self._fallback_index.add_with_ids(batch_vecs, batch_ids)
|
||||
|
||||
self._update_reservoir(batch_vecs)
|
||||
# 这里的 TRAIN_SIZE 取默认 10k，或者根据当前数据量动态判断
|
||||
if len(self._reservoir_buffer) >= 10000:
|
||||
logger.info(f"训练样本达到上限，开始训练...")
|
||||
self._train_and_replay_unlocked()
|
||||
|
||||
self._total_added += len(batch_ids)
|
||||
return len(batch_ids)
|
||||
|
||||
def _flush_write_buffer(self):
|
||||
with self._lock:
|
||||
self._flush_write_buffer_unlocked()
|
||||
|
||||
def _flush_write_buffer_unlocked(self):
|
||||
if not self._write_buffer_vecs:
|
||||
return
|
||||
|
||||
batch_vecs = np.concatenate(self._write_buffer_vecs, axis=0)
|
||||
batch_ids = np.array(self._write_buffer_ids, dtype=np.int64)
|
||||
|
||||
vecs_fp16 = batch_vecs.astype(np.float16)
|
||||
|
||||
with open(self._bin_path, "ab") as f:
|
||||
f.write(vecs_fp16.tobytes())
|
||||
|
||||
ids_bytes = batch_ids.astype('>i8').tobytes()
|
||||
with open(self._ids_bin_path, "ab") as f:
|
||||
f.write(ids_bytes)
|
||||
|
||||
self._bin_count += len(batch_ids)
|
||||
|
||||
if self._is_trained and self._index.is_trained:
|
||||
self._index.add_with_ids(batch_vecs, batch_ids)
|
||||
else:
|
||||
# 即使在 flush 时，如果未训练，也要同步到 fallback
|
||||
self._fallback_index.add_with_ids(batch_vecs, batch_ids)
|
||||
|
||||
self._write_buffer_vecs.clear()
|
||||
self._write_buffer_ids.clear()
|
||||
|
||||
def _update_reservoir(self, vectors: np.ndarray):
|
||||
for vec in vectors:
|
||||
self._seen_count_for_reservoir += 1
|
||||
if len(self._reservoir_buffer) < self.RESERVOIR_CAPACITY:
|
||||
self._reservoir_buffer.append(vec)
|
||||
else:
|
||||
if self._seen_count_for_reservoir <= self.RESERVOIR_SAMPLE_SCOPE:
|
||||
r = random.randint(0, self._seen_count_for_reservoir - 1)
|
||||
if r < self.RESERVOIR_CAPACITY:
|
||||
self._reservoir_buffer[r] = vec
|
||||
|
||||
def _train_and_replay(self):
|
||||
with self._lock:
|
||||
self._train_and_replay_unlocked()
|
||||
|
||||
def _train_and_replay_unlocked(self):
|
||||
if not self._reservoir_buffer:
|
||||
logger.warning("No training data available.")
|
||||
return
|
||||
|
||||
train_data = np.array(self._reservoir_buffer, dtype=np.float32)
|
||||
logger.info(f"Training Index with {len(train_data)} samples...")
|
||||
|
||||
try:
|
||||
self._index.train(train_data)
|
||||
except Exception as e:
|
||||
logger.error(f"SQ8 Training failed: {e}. Staying in fallback mode.")
|
||||
return
|
||||
|
||||
self._is_trained = True
|
||||
self._reservoir_buffer = []
|
||||
|
||||
logger.info("Replaying data from disk to populate index...")
|
||||
try:
|
||||
replay_count = self._replay_vectors_to_index()
|
||||
# 只有当 replay 成功且数据量一致时，才释放回退索引
|
||||
if self._index.ntotal >= self._bin_count:
|
||||
logger.info(f"Replay successful ({self._index.ntotal}/{self._bin_count}). Releasing fallback index.")
|
||||
self._fallback_index.reset()
|
||||
else:
|
||||
logger.warning(f"Replay count mismatch: {self._index.ntotal} vs {self._bin_count}. Keeping fallback index.")
|
||||
except Exception as e:
|
||||
logger.error(f"Replay failed: {e}. Keeping fallback index as backup.")
|
||||
|
||||
def _replay_vectors_to_index(self) -> int:
|
||||
"""从 vectors.bin 读取并添加到 index"""
|
||||
if not self._bin_path.exists() or not self._ids_bin_path.exists():
|
||||
return 0
|
||||
|
||||
vec_item_size = self.dimension * 2
|
||||
id_item_size = 8
|
||||
chunk_size = 10000
|
||||
|
||||
with open(self._bin_path, "rb") as f_vec, open(self._ids_bin_path, "rb") as f_id:
|
||||
while True:
|
||||
vec_data = f_vec.read(chunk_size * vec_item_size)
|
||||
id_data = f_id.read(chunk_size * id_item_size)
|
||||
|
||||
if not vec_data:
|
||||
break
|
||||
|
||||
batch_fp16 = np.frombuffer(vec_data, dtype=np.float16).reshape(-1, self.dimension)
|
||||
batch_fp32 = batch_fp16.astype(np.float32)
|
||||
faiss.normalize_L2(batch_fp32)
|
||||
|
||||
batch_ids = np.frombuffer(id_data, dtype='>i8').astype(np.int64)
|
||||
|
||||
valid_mask = [id_ not in self._deleted_ids for id_ in batch_ids]
|
||||
if not all(valid_mask):
|
||||
batch_fp32 = batch_fp32[valid_mask]
|
||||
batch_ids = batch_ids[valid_mask]
|
||||
|
||||
if len(batch_ids) > 0:
|
||||
self._index.add_with_ids(batch_fp32, batch_ids)
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: np.ndarray,
|
||||
k: int = 10,
|
||||
filter_deleted: bool = True,
|
||||
) -> Tuple[List[str], List[float]]:
|
||||
query_local = np.array(query, dtype=np.float32, order="C", copy=True)
|
||||
if query_local.ndim == 1:
|
||||
got_dim = int(query_local.shape[0])
|
||||
query_local = query_local.reshape(1, -1)
|
||||
elif query_local.ndim == 2:
|
||||
if query_local.shape[0] != 1:
|
||||
raise ValueError(
|
||||
f"query embedding must have shape (D,) or (1, D), got {tuple(query_local.shape)}"
|
||||
)
|
||||
got_dim = int(query_local.shape[1])
|
||||
else:
|
||||
raise ValueError(
|
||||
f"query embedding must have shape (D,) or (1, D), got {tuple(query_local.shape)}"
|
||||
)
|
||||
|
||||
if got_dim != self.dimension:
|
||||
raise ValueError(
|
||||
f"query embedding dimension mismatch: expected={self.dimension} got={got_dim}"
|
||||
)
|
||||
if not np.all(np.isfinite(query_local)):
|
||||
raise ValueError("query embedding contains non-finite values")
|
||||
|
||||
faiss.normalize_L2(query_local)
|
||||
|
||||
# 查询路径仅负责检索，不在此触发训练/回放。
|
||||
# 训练/回放前置到 warmup_index()，并由插件启动阶段触发。
|
||||
# Faiss 索引在并发 search 下可能出现阻塞，这里串行化检索调用保证稳定性。
|
||||
with self._lock:
|
||||
self._flush_write_buffer_unlocked()
|
||||
search_index = self._index if (self._is_trained and self._index.ntotal > 0) else self._fallback_index
|
||||
if search_index.ntotal == 0:
|
||||
logger.warning("Indices are empty. No data to search.")
|
||||
return [], []
|
||||
# 执行检索
|
||||
dists, ids = search_index.search(query_local, k * 2)
|
||||
|
||||
# Faiss search 返回的是 (1, K) 的数组，取第一行
|
||||
dists = dists[0]
|
||||
ids = ids[0]
|
||||
|
||||
results = []
|
||||
for id_val, score in zip(ids, dists):
|
||||
if id_val == -1: continue
|
||||
if filter_deleted and id_val in self._deleted_ids:
|
||||
continue
|
||||
|
||||
str_id = self._int_to_str_map.get(id_val)
|
||||
if str_id:
|
||||
results.append((str_id, float(score)))
|
||||
|
||||
# Sort and trim just in case filtering reduced count
|
||||
results.sort(key=lambda x: x[1], reverse=True)
|
||||
results = results[:k]
|
||||
|
||||
if not results:
|
||||
return [], []
|
||||
|
||||
return [r[0] for r in results], [r[1] for r in results]
|
||||
|
||||
def warmup_index(self, force_train: bool = True) -> Dict[str, Any]:
|
||||
"""
|
||||
预热向量索引（训练/回放前置），避免首个线上查询触发重初始化。
|
||||
|
||||
Args:
|
||||
force_train: 是否在满足阈值时强制训练 SQ8 索引
|
||||
|
||||
Returns:
|
||||
预热状态摘要
|
||||
"""
|
||||
started = time.perf_counter()
|
||||
logger.info(f"metric.vector_index_prewarm_started=1 force_train={bool(force_train)}")
|
||||
|
||||
try:
|
||||
with self._lock:
|
||||
self._flush_write_buffer()
|
||||
|
||||
if self._bin_path.exists():
|
||||
self._bin_count = self._bin_path.stat().st_size // (self.dimension * 2)
|
||||
else:
|
||||
self._bin_count = 0
|
||||
|
||||
needs_fallback_bootstrap = (
|
||||
self._bin_count > 0
|
||||
and self._fallback_index.ntotal == 0
|
||||
and (not self._is_trained or self._index.ntotal == 0)
|
||||
)
|
||||
if needs_fallback_bootstrap:
|
||||
self._bootstrap_fallback_from_disk()
|
||||
|
||||
min_train = max(1, int(getattr(self, "min_train_threshold", self.DEFAULT_MIN_TRAIN)))
|
||||
needs_train = (
|
||||
bool(force_train)
|
||||
and self._bin_count >= min_train
|
||||
and not self._is_trained
|
||||
)
|
||||
if needs_train:
|
||||
self._force_train_small_data()
|
||||
|
||||
duration_ms = (time.perf_counter() - started) * 1000.0
|
||||
summary = {
|
||||
"ok": True,
|
||||
"trained": bool(self._is_trained),
|
||||
"index_ntotal": int(self._index.ntotal),
|
||||
"fallback_ntotal": int(self._fallback_index.ntotal),
|
||||
"bin_count": int(self._bin_count),
|
||||
"duration_ms": duration_ms,
|
||||
"error": None,
|
||||
}
|
||||
except Exception as e:
|
||||
duration_ms = (time.perf_counter() - started) * 1000.0
|
||||
summary = {
|
||||
"ok": False,
|
||||
"trained": bool(self._is_trained),
|
||||
"index_ntotal": int(self._index.ntotal) if self._index is not None else 0,
|
||||
"fallback_ntotal": int(self._fallback_index.ntotal) if self._fallback_index is not None else 0,
|
||||
"bin_count": int(getattr(self, "_bin_count", 0)),
|
||||
"duration_ms": duration_ms,
|
||||
"error": str(e),
|
||||
}
|
||||
logger.error(
|
||||
"metric.vector_index_prewarm_fail=1 "
|
||||
f"metric.vector_index_prewarm_duration_ms={duration_ms:.2f} "
|
||||
f"error={e}"
|
||||
)
|
||||
return summary
|
||||
|
||||
logger.info(
|
||||
"metric.vector_index_prewarm_success=1 "
|
||||
f"metric.vector_index_prewarm_duration_ms={summary['duration_ms']:.2f} "
|
||||
f"trained={summary['trained']} "
|
||||
f"index_ntotal={summary['index_ntotal']} "
|
||||
f"fallback_ntotal={summary['fallback_ntotal']} "
|
||||
f"bin_count={summary['bin_count']}"
|
||||
)
|
||||
return summary
|
||||
|
||||
def _bootstrap_fallback_from_disk(self):
|
||||
with self._lock:
|
||||
self._bootstrap_fallback_from_disk_unlocked()
|
||||
|
||||
def _bootstrap_fallback_from_disk_unlocked(self):
|
||||
"""重启后自举：从磁盘 vectors.bin 加载数据到 fallback 索引"""
|
||||
if not self._bin_path.exists() or not self._ids_bin_path.exists():
|
||||
return
|
||||
|
||||
logger.info("Replaying all disk vectors to fallback index...")
|
||||
vec_item_size = self.dimension * 2
|
||||
id_item_size = 8
|
||||
chunk_size = 10000
|
||||
|
||||
with open(self._bin_path, "rb") as f_vec, open(self._ids_bin_path, "rb") as f_id:
|
||||
while True:
|
||||
vec_data = f_vec.read(chunk_size * vec_item_size)
|
||||
id_data = f_id.read(chunk_size * id_item_size)
|
||||
if not vec_data: break
|
||||
|
||||
batch_fp16 = np.frombuffer(vec_data, dtype=np.float16).reshape(-1, self.dimension)
|
||||
batch_fp32 = batch_fp16.astype(np.float32)
|
||||
faiss.normalize_L2(batch_fp32)
|
||||
batch_ids = np.frombuffer(id_data, dtype='>i8').astype(np.int64)
|
||||
|
||||
valid_mask = [id_ not in self._deleted_ids for id_ in batch_ids]
|
||||
if any(valid_mask):
|
||||
self._fallback_index.add_with_ids(batch_fp32[valid_mask], batch_ids[valid_mask])
|
||||
|
||||
logger.info(f"Fallback index self-bootstrapped with {self._fallback_index.ntotal} items.")
|
||||
|
||||
def _force_train_small_data(self):
|
||||
with self._lock:
|
||||
self._force_train_small_data_unlocked()
|
||||
|
||||
def _force_train_small_data_unlocked(self):
|
||||
logger.info("Forcing training on small dataset...")
|
||||
self._reservoir_buffer = []
|
||||
|
||||
chunk_size = 10000
|
||||
vec_item_size = self.dimension * 2
|
||||
|
||||
with open(self._bin_path, "rb") as f:
|
||||
while len(self._reservoir_buffer) < self.TRAIN_SIZE:
|
||||
data = f.read(chunk_size * vec_item_size)
|
||||
if not data: break
|
||||
fp16 = np.frombuffer(data, dtype=np.float16).reshape(-1, self.dimension)
|
||||
fp32 = fp16.astype(np.float32)
|
||||
faiss.normalize_L2(fp32)
|
||||
|
||||
for vec in fp32:
|
||||
self._reservoir_buffer.append(vec)
|
||||
if len(self._reservoir_buffer) >= self.TRAIN_SIZE:
|
||||
break
|
||||
|
||||
self._train_and_replay_unlocked()
|
||||
|
||||
def delete(self, ids: List[str]) -> int:
|
||||
with self._lock:
|
||||
count = 0
|
||||
for str_id in ids:
|
||||
if str_id not in self._known_hashes:
|
||||
continue
|
||||
int_id = self._generate_id(str_id)
|
||||
if int_id not in self._deleted_ids:
|
||||
self._deleted_ids.add(int_id)
|
||||
if self._index.is_trained:
|
||||
self._index.remove_ids(np.array([int_id], dtype=np.int64))
|
||||
# 同步从 fallback 移除
|
||||
if self._fallback_index.ntotal > 0:
|
||||
self._fallback_index.remove_ids(np.array([int_id], dtype=np.int64))
|
||||
count += 1
|
||||
self._total_deleted += count
|
||||
|
||||
# Check GC
|
||||
self._check_rebuild_needed()
|
||||
return count
|
||||
|
||||
def _check_rebuild_needed(self):
|
||||
"""GC Excution Check"""
|
||||
if self._bin_count == 0: return
|
||||
ratio = len(self._deleted_ids) / self._bin_count
|
||||
if ratio > 0.3 and len(self._deleted_ids) > 1000:
|
||||
logger.info(f"Triggering GC/Rebuild (deleted ratio: {ratio:.2f})")
|
||||
self.rebuild_index()
|
||||
|
||||
def rebuild_index(self):
|
||||
"""GC: 重建索引，压缩 bin 文件"""
|
||||
with self._lock:
|
||||
self._rebuild_index_locked()
|
||||
|
||||
def _rebuild_index_locked(self):
|
||||
"""实际 GC 重建逻辑。"""
|
||||
logger.info("Starting Compaction (GC)...")
|
||||
|
||||
tmp_bin = self.data_dir / "vectors.bin.tmp"
|
||||
tmp_ids = self.data_dir / "vectors_ids.bin.tmp"
|
||||
|
||||
vec_item_size = self.dimension * 2
|
||||
id_item_size = 8
|
||||
chunk_size = 10000
|
||||
|
||||
new_count = 0
|
||||
|
||||
# 1. Compact Files
|
||||
with open(self._bin_path, "rb") as f_vec, open(self._ids_bin_path, "rb") as f_id, \
|
||||
open(tmp_bin, "wb") as w_vec, open(tmp_ids, "wb") as w_id:
|
||||
while True:
|
||||
vec_data = f_vec.read(chunk_size * vec_item_size)
|
||||
id_data = f_id.read(chunk_size * id_item_size)
|
||||
if not vec_data: break
|
||||
|
||||
batch_fp16 = np.frombuffer(vec_data, dtype=np.float16).reshape(-1, self.dimension)
|
||||
batch_ids = np.frombuffer(id_data, dtype='>i8').astype(np.int64)
|
||||
|
||||
keep_mask = [id_ not in self._deleted_ids for id_ in batch_ids]
|
||||
|
||||
if any(keep_mask):
|
||||
keep_vecs = batch_fp16[keep_mask]
|
||||
keep_ids = batch_ids[keep_mask]
|
||||
|
||||
w_vec.write(keep_vecs.tobytes())
|
||||
w_id.write(keep_ids.astype('>i8').tobytes())
|
||||
new_count += len(keep_ids)
|
||||
|
||||
# 2. Reset State & Atomic Swap
|
||||
self._bin_count = new_count
|
||||
|
||||
# Close current index
|
||||
self._index.reset()
|
||||
if self._fallback_index: self._fallback_index.reset() # Also clear fallback
|
||||
self._is_trained = False
|
||||
|
||||
# Swap files
|
||||
shutil.move(str(tmp_bin), str(self._bin_path))
|
||||
shutil.move(str(tmp_ids), str(self._ids_bin_path))
|
||||
|
||||
# Reset Tombstones (Critical)
|
||||
self._deleted_ids.clear()
|
||||
|
||||
# 3. Reload/Rebuild Index (Fresh Train)
|
||||
# We need to re-train because data distribution might have changed significantly after deletion
|
||||
self._init_index()
|
||||
self._init_fallback_index() # Re-init fallback too
|
||||
self._force_train_small_data() # This will train and replay from the NEW compact file
|
||||
|
||||
logger.info("Compaction Complete.")
|
||||
|
||||
def save(self, data_dir: Optional[Union[str, Path]] = None) -> None:
|
||||
with self._lock:
|
||||
if not data_dir:
|
||||
data_dir = self.data_dir
|
||||
if not data_dir:
|
||||
raise ValueError("No data_dir")
|
||||
|
||||
data_dir = Path(data_dir)
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._flush_write_buffer_unlocked()
|
||||
|
||||
if self._is_trained:
|
||||
index_path = data_dir / "vectors.index"
|
||||
with atomic_save_path(index_path) as tmp:
|
||||
faiss.write_index(self._index, tmp)
|
||||
|
||||
meta = {
|
||||
"dimension": self.dimension,
|
||||
"quantization_type": self.quantization_type.value,
|
||||
"is_trained": self._is_trained,
|
||||
"vector_norm": self._vector_norm,
|
||||
"deleted_ids": list(self._deleted_ids),
|
||||
"known_hashes": list(self._known_hashes),
|
||||
}
|
||||
|
||||
with atomic_write(data_dir / "vectors_metadata.pkl", "wb") as f:
|
||||
pickle.dump(meta, f)
|
||||
|
||||
logger.info("VectorStore saved.")
|
||||
|
||||
def migrate_legacy_npy(self, data_dir: Optional[Union[str, Path]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
离线迁移入口：将 legacy vectors.npy 转为 vNext 二进制格式。
|
||||
"""
|
||||
with self._lock:
|
||||
target_dir = Path(data_dir) if data_dir else self.data_dir
|
||||
if target_dir is None:
|
||||
raise ValueError("No data_dir")
|
||||
target_dir = Path(target_dir)
|
||||
npy_path = target_dir / "vectors.npy"
|
||||
idx_path = target_dir / "vectors.index"
|
||||
bin_path = target_dir / "vectors.bin"
|
||||
ids_bin_path = target_dir / "vectors_ids.bin"
|
||||
meta_path = target_dir / "vectors_metadata.pkl"
|
||||
|
||||
if not npy_path.exists():
|
||||
return {"migrated": False, "reason": "npy_missing"}
|
||||
if not meta_path.exists():
|
||||
raise RuntimeError("legacy vectors.npy migration requires vectors_metadata.pkl")
|
||||
if bin_path.exists() and ids_bin_path.exists():
|
||||
return {"migrated": False, "reason": "bin_exists"}
|
||||
|
||||
# Reset in-memory state to avoid appending to stale runtime buffers.
|
||||
self._known_hashes.clear()
|
||||
self._deleted_ids.clear()
|
||||
self._write_buffer_vecs.clear()
|
||||
self._write_buffer_ids.clear()
|
||||
self._init_index()
|
||||
self._init_fallback_index()
|
||||
self._is_trained = False
|
||||
self._bin_count = 0
|
||||
|
||||
self._migrate_from_npy_unlocked(npy_path, idx_path, target_dir)
|
||||
self.save(target_dir)
|
||||
return {"migrated": True, "reason": "ok"}
|
||||
|
||||
def load(self, data_dir: Optional[Union[str, Path]] = None) -> None:
|
||||
with self._lock:
|
||||
if not data_dir: data_dir = self.data_dir
|
||||
data_dir = Path(data_dir)
|
||||
|
||||
npy_path = data_dir / "vectors.npy"
|
||||
idx_path = data_dir / "vectors.index"
|
||||
bin_path = data_dir / "vectors.bin"
|
||||
|
||||
if npy_path.exists() and not bin_path.exists():
|
||||
raise RuntimeError(
|
||||
"检测到 legacy vectors.npy，vNext 不再支持运行时自动迁移。"
|
||||
" 请先执行 scripts/release_vnext_migrate.py migrate。"
|
||||
)
|
||||
|
||||
meta_path = data_dir / "vectors_metadata.pkl"
|
||||
if not meta_path.exists():
|
||||
logger.warning("No metadata found, initialized empty.")
|
||||
return
|
||||
|
||||
with open(meta_path, "rb") as f:
|
||||
meta = pickle.load(f)
|
||||
|
||||
if meta.get("vector_norm") != "l2":
|
||||
logger.warning("Index IDMap2 version mismatch (L2 Norm), forcing rebuild...")
|
||||
self._known_hashes = set(meta.get("ids", [])) | set(meta.get("known_hashes", []))
|
||||
self._deleted_ids = set(meta.get("deleted_ids", []))
|
||||
self._init_index()
|
||||
self._force_train_small_data()
|
||||
return
|
||||
|
||||
self._is_trained = meta.get("is_trained", False)
|
||||
self._vector_norm = meta.get("vector_norm", "l2")
|
||||
self._deleted_ids = set(meta.get("deleted_ids", []))
|
||||
self._known_hashes = set(meta.get("known_hashes", []))
|
||||
|
||||
if self._is_trained:
|
||||
if idx_path.exists():
|
||||
try:
|
||||
self._index = faiss.read_index(str(idx_path))
|
||||
if not isinstance(self._index, faiss.IndexIDMap2):
|
||||
logger.warning("Loaded index type mismatch. Rebuilding...")
|
||||
self._init_index()
|
||||
self._force_train_small_data()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load index: {e}. Rebuilding...")
|
||||
self._init_index()
|
||||
self._force_train_small_data()
|
||||
else:
|
||||
logger.warning("Index file missing despite metadata indicating trained. Rebuilding from bin...")
|
||||
self._init_index()
|
||||
self._force_train_small_data()
|
||||
|
||||
if bin_path.exists():
|
||||
self._bin_count = bin_path.stat().st_size // (self.dimension * 2)
|
||||
|
||||
def _migrate_from_npy(self, npy_path, idx_path, data_dir):
|
||||
with self._lock:
|
||||
self._migrate_from_npy_unlocked(npy_path, idx_path, data_dir)
|
||||
|
||||
def _migrate_from_npy_unlocked(self, npy_path, idx_path, data_dir):
|
||||
try:
|
||||
arr = np.load(npy_path, mmap_mode="r")
|
||||
except Exception:
|
||||
arr = np.load(npy_path)
|
||||
|
||||
meta_path = data_dir / "vectors_metadata.pkl"
|
||||
old_ids = []
|
||||
if meta_path.exists():
|
||||
with open(meta_path, "rb") as f:
|
||||
m = pickle.load(f)
|
||||
old_ids = m.get("ids", [])
|
||||
|
||||
if len(arr) != len(old_ids):
|
||||
logger.error(f"Migration mismatch: arr {len(arr)} != ids {len(old_ids)}")
|
||||
return
|
||||
|
||||
logger.info(f"Migrating {len(arr)} vectors...")
|
||||
|
||||
chunk = 1000
|
||||
for i in range(0, len(arr), chunk):
|
||||
sub_arr = arr[i : i+chunk]
|
||||
sub_ids = old_ids[i : i+chunk]
|
||||
self.add(sub_arr, sub_ids)
|
||||
|
||||
if not self._is_trained:
|
||||
self._force_train_small_data()
|
||||
|
||||
shutil.move(str(npy_path), str(npy_path) + ".bak")
|
||||
if idx_path.exists():
|
||||
shutil.move(str(idx_path), str(idx_path) + ".bak")
|
||||
|
||||
logger.info("Migration complete.")
|
||||
|
||||
def clear(self) -> None:
|
||||
with self._lock:
|
||||
self._ids_bin_path.unlink(missing_ok=True)
|
||||
self._bin_path.unlink(missing_ok=True)
|
||||
self._init_index()
|
||||
self._known_hashes.clear()
|
||||
self._deleted_ids.clear()
|
||||
self._bin_count = 0
|
||||
logger.info("VectorStore cleared.")
|
||||
|
||||
def has_data(self) -> bool:
|
||||
return (self.data_dir / "vectors_metadata.pkl").exists()
|
||||
|
||||
@property
|
||||
def num_vectors(self) -> int:
|
||||
return len(self._known_hashes) - len(self._deleted_ids)
|
||||
|
||||
def __contains__(self, hash_value: str) -> bool:
|
||||
"""Check if a hash exists in the store"""
|
||||
return hash_value in self._known_hashes and self._generate_id(hash_value) not in self._deleted_ids
|
||||
|
||||
0
src/A_memorix/core/strategies/__init__.py
Normal file
0
src/A_memorix/core/strategies/__init__.py
Normal file
89
src/A_memorix/core/strategies/base.py
Normal file
89
src/A_memorix/core/strategies/base.py
Normal file
@@ -0,0 +1,89 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Dict, Any, Optional, Union
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
import hashlib
|
||||
|
||||
class KnowledgeType(str, Enum):
|
||||
NARRATIVE = "narrative"
|
||||
FACTUAL = "factual"
|
||||
QUOTE = "quote"
|
||||
MIXED = "mixed"
|
||||
|
||||
@dataclass
|
||||
class SourceInfo:
|
||||
file: str
|
||||
offset_start: int
|
||||
offset_end: int
|
||||
checksum: str = ""
|
||||
|
||||
@dataclass
|
||||
class ChunkContext:
|
||||
chunk_id: str
|
||||
index: int
|
||||
context: Dict[str, Any] = field(default_factory=dict)
|
||||
text: str = ""
|
||||
|
||||
@dataclass
|
||||
class ChunkFlags:
|
||||
verbatim: bool = False
|
||||
requires_llm: bool = True
|
||||
|
||||
@dataclass
|
||||
class ProcessedChunk:
|
||||
type: KnowledgeType
|
||||
source: SourceInfo
|
||||
chunk: ChunkContext
|
||||
data: Dict[str, Any] = field(default_factory=dict) # triples、events、verbatim_entities
|
||||
flags: ChunkFlags = field(default_factory=ChunkFlags)
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
return {
|
||||
"type": self.type.value,
|
||||
"source": {
|
||||
"file": self.source.file,
|
||||
"offset_start": self.source.offset_start,
|
||||
"offset_end": self.source.offset_end,
|
||||
"checksum": self.source.checksum
|
||||
},
|
||||
"chunk": {
|
||||
"text": self.chunk.text,
|
||||
"chunk_id": self.chunk.chunk_id,
|
||||
"index": self.chunk.index,
|
||||
"context": self.chunk.context
|
||||
},
|
||||
"data": self.data,
|
||||
"flags": {
|
||||
"verbatim": self.flags.verbatim,
|
||||
"requires_llm": self.flags.requires_llm
|
||||
}
|
||||
}
|
||||
|
||||
class BaseStrategy(ABC):
|
||||
def __init__(self, filename: str):
|
||||
self.filename = filename
|
||||
|
||||
@abstractmethod
|
||||
def split(self, text: str) -> List[ProcessedChunk]:
|
||||
"""按策略将文本切分为块。"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def extract(self, chunk: ProcessedChunk, llm_func=None) -> ProcessedChunk:
|
||||
"""从文本块中抽取结构化信息。"""
|
||||
pass
|
||||
|
||||
def calculate_checksum(self, text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
def build_language_guard(self, text: str) -> str:
|
||||
"""
|
||||
构建统一的输出语言约束。
|
||||
不区分语言类型，仅要求抽取值保持原文语言，不做翻译。
|
||||
"""
|
||||
_ = text # 预留参数，便于后续按需扩展
|
||||
return (
|
||||
"Focus on the original source language. Keep extracted events, entities, predicates "
|
||||
"and objects in the same language as the source text, preserve names/terms as-is, "
|
||||
"and do not translate."
|
||||
)
|
||||
98
src/A_memorix/core/strategies/factual.py
Normal file
98
src/A_memorix/core/strategies/factual.py
Normal file
@@ -0,0 +1,98 @@
|
||||
import re
|
||||
from typing import List, Dict, Any
|
||||
from .base import BaseStrategy, ProcessedChunk, KnowledgeType, SourceInfo, ChunkContext
|
||||
|
||||
class FactualStrategy(BaseStrategy):
|
||||
def split(self, text: str) -> List[ProcessedChunk]:
|
||||
# 结构感知切分
|
||||
lines = text.split('\n')
|
||||
chunks = []
|
||||
current_chunk_lines = []
|
||||
current_len = 0
|
||||
target_size = 600
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
# 判断是否应当切分
|
||||
# 若当前行为列表项/定义/表格行，则尽量不切分
|
||||
is_structure = self._is_structural_line(line)
|
||||
|
||||
current_len += len(line) + 1
|
||||
current_chunk_lines.append(line)
|
||||
|
||||
# 达到目标长度且不在紧凑结构块内时切分（过长时强制切分）
|
||||
if current_len >= target_size and not is_structure:
|
||||
chunks.append(self._create_chunk(current_chunk_lines, len(chunks)))
|
||||
current_chunk_lines = []
|
||||
current_len = 0
|
||||
elif current_len >= target_size * 2: # 超长时强制切分
|
||||
chunks.append(self._create_chunk(current_chunk_lines, len(chunks)))
|
||||
current_chunk_lines = []
|
||||
current_len = 0
|
||||
|
||||
if current_chunk_lines:
|
||||
chunks.append(self._create_chunk(current_chunk_lines, len(chunks)))
|
||||
|
||||
return chunks
|
||||
|
||||
def _is_structural_line(self, line: str) -> bool:
|
||||
line = line.strip()
|
||||
if not line: return False
|
||||
# 列表项
|
||||
if re.match(r'^[\-\*]\s+', line) or re.match(r'^\d+\.\s+', line):
|
||||
return True
|
||||
# 定义项（术语: 定义）
|
||||
if re.match(r'^[^：:]+[：:].+', line):
|
||||
return True
|
||||
# 表格行（按 markdown 语法假设）
|
||||
if line.startswith('|') and line.endswith('|'):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _create_chunk(self, lines: List[str], index: int) -> ProcessedChunk:
|
||||
text = "\n".join(lines)
|
||||
return ProcessedChunk(
|
||||
type=KnowledgeType.FACTUAL,
|
||||
source=SourceInfo(
|
||||
file=self.filename,
|
||||
offset_start=0, # 简化处理：真实偏移跟踪需要额外状态
|
||||
offset_end=0,
|
||||
checksum=self.calculate_checksum(text)
|
||||
),
|
||||
chunk=ChunkContext(
|
||||
chunk_id=f"{self.filename}_{index}",
|
||||
index=index,
|
||||
text=text
|
||||
)
|
||||
)
|
||||
|
||||
async def extract(self, chunk: ProcessedChunk, llm_func=None) -> ProcessedChunk:
|
||||
if not llm_func:
|
||||
raise ValueError("LLM function required for Factual extraction")
|
||||
|
||||
language_guard = self.build_language_guard(chunk.chunk.text)
|
||||
prompt = f"""You are a factual knowledge extraction engine.
|
||||
Extract factual triples and entities from the text.
|
||||
Preserve lists and definitions accurately.
|
||||
|
||||
Language constraints:
|
||||
- {language_guard}
|
||||
- Preserve original names and domain terms exactly when possible.
|
||||
- JSON keys must stay exactly as: triples, entities, subject, predicate, object.
|
||||
|
||||
Text:
|
||||
{chunk.chunk.text}
|
||||
|
||||
Return ONLY valid JSON:
|
||||
{{
|
||||
"triples": [
|
||||
{{"subject": "Entity", "predicate": "Relationship", "object": "Entity"}}
|
||||
],
|
||||
"entities": ["Entity1", "Entity2"]
|
||||
}}
|
||||
"""
|
||||
result = await llm_func(prompt)
|
||||
|
||||
# 结果保持原样存入 data，后续统一归一化流程会处理
|
||||
# vector_store 侧期望关系字段为 subject/predicate/object 映射形式
|
||||
chunk.data = result
|
||||
return chunk
|
||||
126
src/A_memorix/core/strategies/narrative.py
Normal file
126
src/A_memorix/core/strategies/narrative.py
Normal file
@@ -0,0 +1,126 @@
|
||||
import re
|
||||
from typing import List, Dict, Any
|
||||
from .base import BaseStrategy, ProcessedChunk, KnowledgeType, SourceInfo, ChunkContext
|
||||
|
||||
class NarrativeStrategy(BaseStrategy):
|
||||
def split(self, text: str) -> List[ProcessedChunk]:
|
||||
scenes = self._split_into_scenes(text)
|
||||
chunks = []
|
||||
|
||||
for scene_idx, (scene_text, scene_title) in enumerate(scenes):
|
||||
scene_chunks = self._sliding_window(scene_text, scene_title, scene_idx)
|
||||
chunks.extend(scene_chunks)
|
||||
|
||||
return chunks
|
||||
|
||||
def _split_into_scenes(self, text: str) -> List[tuple[str, str]]:
|
||||
"""按标题或分隔符把文本切分为场景。"""
|
||||
# 简单启发式：按 markdown 标题或特定分隔符切分
|
||||
# 该正则匹配以 #、Chapter 或 *** / === 开头的分隔行
|
||||
# 该正则匹配以 #、Chapter 或 *** / === 开头的分隔行
|
||||
scene_pattern_str = r'^(?:#{1,6}\s+.*|Chapter\s+\d+|^\*{3,}$|^={3,}$)'
|
||||
|
||||
# 保留分隔符，以便识别场景起点
|
||||
parts = re.split(f"({scene_pattern_str})", text, flags=re.MULTILINE)
|
||||
|
||||
scenes = []
|
||||
current_scene_title = "Start"
|
||||
current_scene_content = []
|
||||
|
||||
if parts and parts[0].strip() == "":
|
||||
parts = parts[1:]
|
||||
|
||||
for part in parts:
|
||||
if re.match(scene_pattern_str, part, re.MULTILINE):
|
||||
# 先保存上一段场景
|
||||
if current_scene_content:
|
||||
scenes.append(("".join(current_scene_content), current_scene_title))
|
||||
current_scene_content = []
|
||||
current_scene_title = part.strip()
|
||||
else:
|
||||
current_scene_content.append(part)
|
||||
|
||||
if current_scene_content:
|
||||
scenes.append(("".join(current_scene_content), current_scene_title))
|
||||
|
||||
# 若未识别到场景，则把全文视作单一场景
|
||||
if not scenes:
|
||||
scenes = [(text, "Whole Text")]
|
||||
|
||||
return scenes
|
||||
|
||||
def _sliding_window(self, text: str, scene_id: str, scene_idx: int, window_size=800, overlap=200) -> List[ProcessedChunk]:
|
||||
chunks = []
|
||||
if len(text) <= window_size:
|
||||
chunks.append(self._create_chunk(text, scene_id, scene_idx, 0, 0))
|
||||
return chunks
|
||||
|
||||
stride = window_size - overlap
|
||||
start = 0
|
||||
local_idx = 0
|
||||
while start < len(text):
|
||||
end = min(start + window_size, len(text))
|
||||
chunk_text = text[start:end]
|
||||
|
||||
# 尽量对齐到最近换行，避免生硬截断句子
|
||||
# 仅在未到文本尾部时进行回退
|
||||
if end < len(text):
|
||||
last_newline = chunk_text.rfind('\n')
|
||||
if last_newline > window_size // 2: # 仅在回退距离可接受时启用
|
||||
end = start + last_newline + 1
|
||||
chunk_text = text[start:end]
|
||||
|
||||
chunks.append(self._create_chunk(chunk_text, scene_id, scene_idx, local_idx, start))
|
||||
|
||||
start += len(chunk_text) - overlap if end < len(text) else len(chunk_text)
|
||||
local_idx += 1
|
||||
|
||||
return chunks
|
||||
|
||||
def _create_chunk(self, text: str, scene_id: str, scene_idx: int, local_idx: int, offset: int) -> ProcessedChunk:
|
||||
return ProcessedChunk(
|
||||
type=KnowledgeType.NARRATIVE,
|
||||
source=SourceInfo(
|
||||
file=self.filename,
|
||||
offset_start=offset,
|
||||
offset_end=offset + len(text),
|
||||
checksum=self.calculate_checksum(text)
|
||||
),
|
||||
chunk=ChunkContext(
|
||||
chunk_id=f"{self.filename}_{scene_idx}_{local_idx}",
|
||||
index=local_idx,
|
||||
text=text,
|
||||
context={"scene_id": scene_id}
|
||||
)
|
||||
)
|
||||
|
||||
async def extract(self, chunk: ProcessedChunk, llm_func=None) -> ProcessedChunk:
|
||||
if not llm_func:
|
||||
raise ValueError("LLM function required for Narrative extraction")
|
||||
|
||||
language_guard = self.build_language_guard(chunk.chunk.text)
|
||||
prompt = f"""You are a narrative knowledge extraction engine.
|
||||
Extract key events and character relations from the scene text.
|
||||
|
||||
Language constraints:
|
||||
- {language_guard}
|
||||
- Preserve original names and terms exactly when possible.
|
||||
- JSON keys must stay exactly as: events, relations, subject, predicate, object.
|
||||
|
||||
Scene:
|
||||
{chunk.chunk.context.get('scene_id')}
|
||||
|
||||
Text:
|
||||
{chunk.chunk.text}
|
||||
|
||||
Return ONLY valid JSON:
|
||||
{{
|
||||
"events": ["event description 1", "event description 2"],
|
||||
"relations": [
|
||||
{{"subject": "CharacterA", "predicate": "relation", "object": "CharacterB"}}
|
||||
]
|
||||
}}
|
||||
"""
|
||||
result = await llm_func(prompt)
|
||||
chunk.data = result
|
||||
return chunk
|
||||
52
src/A_memorix/core/strategies/quote.py
Normal file
52
src/A_memorix/core/strategies/quote.py
Normal file
@@ -0,0 +1,52 @@
|
||||
from typing import List, Dict, Any
|
||||
from .base import BaseStrategy, ProcessedChunk, KnowledgeType, SourceInfo, ChunkContext, ChunkFlags
|
||||
|
||||
class QuoteStrategy(BaseStrategy):
|
||||
def split(self, text: str) -> List[ProcessedChunk]:
|
||||
# Split by double newlines (stanzas)
|
||||
stanzas = text.split("\n\n")
|
||||
chunks = []
|
||||
offset = 0
|
||||
|
||||
for idx, stanza in enumerate(stanzas):
|
||||
if not stanza.strip():
|
||||
offset += len(stanza) + 2
|
||||
continue
|
||||
|
||||
chunk = ProcessedChunk(
|
||||
type=KnowledgeType.QUOTE,
|
||||
source=SourceInfo(
|
||||
file=self.filename,
|
||||
offset_start=offset,
|
||||
offset_end=offset + len(stanza),
|
||||
checksum=self.calculate_checksum(stanza)
|
||||
),
|
||||
chunk=ChunkContext(
|
||||
chunk_id=f"{self.filename}_{idx}",
|
||||
index=idx,
|
||||
text=stanza
|
||||
),
|
||||
flags=ChunkFlags(
|
||||
verbatim=True,
|
||||
requires_llm=False # Default to no LLM, but can be overridden
|
||||
)
|
||||
)
|
||||
chunks.append(chunk)
|
||||
offset += len(stanza) + 2 # +2 for \n\n
|
||||
|
||||
return chunks
|
||||
|
||||
async def extract(self, chunk: ProcessedChunk, llm_func=None) -> ProcessedChunk:
|
||||
# For quotes, the text itself is the entity/knowledge
|
||||
# We might use LLM to extract headers/metadata if requested, but core logic is pass-through
|
||||
|
||||
# Treat the whole chunk text as a verbatim entity
|
||||
chunk.data = {
|
||||
"verbatim_entities": [chunk.chunk.text]
|
||||
}
|
||||
|
||||
if llm_func and chunk.flags.requires_llm:
|
||||
# Optional: Extract metadata
|
||||
pass
|
||||
|
||||
return chunk
|
||||
33
src/A_memorix/core/utils/__init__.py
Normal file
33
src/A_memorix/core/utils/__init__.py
Normal file
@@ -0,0 +1,33 @@
|
||||
"""工具模块 - 哈希、监控等辅助功能"""
|
||||
|
||||
from .hash import compute_hash, normalize_text
|
||||
from .monitor import MemoryMonitor
|
||||
from .quantization import quantize_vector, dequantize_vector
|
||||
from .time_parser import (
|
||||
parse_query_datetime_to_timestamp,
|
||||
parse_query_time_range,
|
||||
parse_ingest_datetime_to_timestamp,
|
||||
normalize_time_meta,
|
||||
format_timestamp,
|
||||
)
|
||||
from .relation_write_service import RelationWriteService, RelationWriteResult
|
||||
from .relation_query import RelationQuerySpec, parse_relation_query_spec
|
||||
from .plugin_id_policy import PluginIdPolicy
|
||||
|
||||
__all__ = [
|
||||
"compute_hash",
|
||||
"normalize_text",
|
||||
"MemoryMonitor",
|
||||
"quantize_vector",
|
||||
"dequantize_vector",
|
||||
"parse_query_datetime_to_timestamp",
|
||||
"parse_query_time_range",
|
||||
"parse_ingest_datetime_to_timestamp",
|
||||
"normalize_time_meta",
|
||||
"format_timestamp",
|
||||
"RelationWriteService",
|
||||
"RelationWriteResult",
|
||||
"RelationQuerySpec",
|
||||
"parse_relation_query_spec",
|
||||
"PluginIdPolicy",
|
||||
]
|
||||
360
src/A_memorix/core/utils/aggregate_query_service.py
Normal file
360
src/A_memorix/core/utils/aggregate_query_service.py
Normal file
@@ -0,0 +1,360 @@
|
||||
"""
|
||||
聚合查询服务：
|
||||
- 并发执行 search/time/episode 分支
|
||||
- 统一分支结果结构
|
||||
- 可选混合排序（Weighted RRF）
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from src.common.logger import get_logger
|
||||
|
||||
logger = get_logger("A_Memorix.AggregateQueryService")
|
||||
|
||||
BranchRunner = Callable[[], Awaitable[Dict[str, Any]]]
|
||||
|
||||
|
||||
class AggregateQueryService:
|
||||
"""聚合查询执行服务（search/time/episode）。"""
|
||||
|
||||
def __init__(self, plugin_config: Optional[Any] = None):
|
||||
self.plugin_config = plugin_config or {}
|
||||
|
||||
def _cfg(self, key: str, default: Any = None) -> Any:
|
||||
getter = getattr(self.plugin_config, "get_config", None)
|
||||
if callable(getter):
|
||||
return getter(key, default)
|
||||
|
||||
current: Any = self.plugin_config
|
||||
for part in key.split("."):
|
||||
if isinstance(current, dict) and part in current:
|
||||
current = current[part]
|
||||
else:
|
||||
return default
|
||||
return current
|
||||
|
||||
@staticmethod
|
||||
def _as_float(value: Any, default: float = 0.0) -> float:
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return float(default)
|
||||
|
||||
@staticmethod
|
||||
def _as_int(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except Exception:
|
||||
return int(default)
|
||||
|
||||
def _rrf_k(self) -> float:
|
||||
raw = self._cfg("retrieval.aggregate.rrf_k", 60.0)
|
||||
value = self._as_float(raw, 60.0)
|
||||
return max(1.0, value)
|
||||
|
||||
def _weights(self) -> Dict[str, float]:
|
||||
defaults = {"search": 1.0, "time": 1.0, "episode": 1.0}
|
||||
raw = self._cfg("retrieval.aggregate.weights", {})
|
||||
if not isinstance(raw, dict):
|
||||
return defaults
|
||||
|
||||
out = dict(defaults)
|
||||
for key in ("search", "time", "episode"):
|
||||
if key in raw:
|
||||
out[key] = max(0.0, self._as_float(raw.get(key), defaults[key]))
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _normalize_branch_payload(
|
||||
name: str,
|
||||
payload: Optional[Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
data = payload if isinstance(payload, dict) else {}
|
||||
results_raw = data.get("results", [])
|
||||
results = results_raw if isinstance(results_raw, list) else []
|
||||
count = data.get("count")
|
||||
if count is None:
|
||||
count = len(results)
|
||||
return {
|
||||
"name": name,
|
||||
"success": bool(data.get("success", False)),
|
||||
"skipped": bool(data.get("skipped", False)),
|
||||
"skip_reason": str(data.get("skip_reason", "") or "").strip(),
|
||||
"error": str(data.get("error", "") or "").strip(),
|
||||
"results": results,
|
||||
"count": max(0, int(count)),
|
||||
"elapsed_ms": max(0.0, float(data.get("elapsed_ms", 0.0) or 0.0)),
|
||||
"content": str(data.get("content", "") or ""),
|
||||
"query_type": str(data.get("query_type", "") or name),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _mix_key(item: Dict[str, Any], branch: str, rank: int) -> str:
|
||||
item_type = str(item.get("type", "") or "").strip().lower()
|
||||
if item_type == "episode":
|
||||
episode_id = str(item.get("episode_id", "") or "").strip()
|
||||
if episode_id:
|
||||
return f"episode:{episode_id}"
|
||||
|
||||
item_hash = str(item.get("hash", "") or "").strip()
|
||||
if item_hash:
|
||||
return f"{item_type}:{item_hash}"
|
||||
|
||||
return f"{branch}:{item_type}:{rank}:{str(item.get('content', '') or '')[:80]}"
|
||||
|
||||
def _build_mixed_results(
|
||||
self,
|
||||
*,
|
||||
branches: Dict[str, Dict[str, Any]],
|
||||
top_k: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
rrf_k = self._rrf_k()
|
||||
weights = self._weights()
|
||||
bucket: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
for branch_name, branch in branches.items():
|
||||
if not branch.get("success", False):
|
||||
continue
|
||||
results = branch.get("results", [])
|
||||
if not isinstance(results, list):
|
||||
continue
|
||||
|
||||
weight = max(0.0, float(weights.get(branch_name, 1.0)))
|
||||
for idx, item in enumerate(results, start=1):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
key = self._mix_key(item, branch_name, idx)
|
||||
score = weight / (rrf_k + float(idx))
|
||||
if key not in bucket:
|
||||
merged = dict(item)
|
||||
merged["fusion_score"] = 0.0
|
||||
merged["_source_branches"] = set()
|
||||
bucket[key] = merged
|
||||
|
||||
target = bucket[key]
|
||||
target["fusion_score"] = float(target.get("fusion_score", 0.0)) + score
|
||||
target["_source_branches"].add(branch_name)
|
||||
|
||||
mixed = list(bucket.values())
|
||||
mixed.sort(
|
||||
key=lambda x: (
|
||||
-float(x.get("fusion_score", 0.0)),
|
||||
str(x.get("type", "") or ""),
|
||||
str(x.get("hash", "") or x.get("episode_id", "") or ""),
|
||||
)
|
||||
)
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
for rank, item in enumerate(mixed[: max(1, int(top_k))], start=1):
|
||||
merged = dict(item)
|
||||
branches_set = merged.pop("_source_branches", set())
|
||||
merged["source_branches"] = sorted(list(branches_set))
|
||||
merged["rank"] = rank
|
||||
out.append(merged)
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _status(branch: Dict[str, Any]) -> str:
|
||||
if branch.get("skipped", False):
|
||||
return "skipped"
|
||||
if branch.get("success", False):
|
||||
return "success"
|
||||
return "failed"
|
||||
|
||||
def _build_summary(self, branches: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
|
||||
summary: Dict[str, Dict[str, Any]] = {}
|
||||
for name, branch in branches.items():
|
||||
status = self._status(branch)
|
||||
summary[name] = {
|
||||
"status": status,
|
||||
"count": int(branch.get("count", 0) or 0),
|
||||
}
|
||||
if status == "skipped":
|
||||
summary[name]["reason"] = str(branch.get("skip_reason", "") or "")
|
||||
if status == "failed":
|
||||
summary[name]["error"] = str(branch.get("error", "") or "")
|
||||
return summary
|
||||
|
||||
def _build_content(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
branches: Dict[str, Dict[str, Any]],
|
||||
errors: List[Dict[str, str]],
|
||||
mixed_results: Optional[List[Dict[str, Any]]],
|
||||
) -> str:
|
||||
lines: List[str] = [
|
||||
f"🔀 聚合查询结果（query='{query or 'N/A'}'）",
|
||||
"",
|
||||
"分支状态：",
|
||||
]
|
||||
for name in ("search", "time", "episode"):
|
||||
branch = branches.get(name, {})
|
||||
status = self._status(branch)
|
||||
count = int(branch.get("count", 0) or 0)
|
||||
line = f"- {name}: {status}, count={count}"
|
||||
reason = str(branch.get("skip_reason", "") or "").strip()
|
||||
err = str(branch.get("error", "") or "").strip()
|
||||
if status == "skipped" and reason:
|
||||
line += f" ({reason})"
|
||||
if status == "failed" and err:
|
||||
line += f" ({err})"
|
||||
lines.append(line)
|
||||
|
||||
if errors:
|
||||
lines.append("")
|
||||
lines.append("错误：")
|
||||
for item in errors[:6]:
|
||||
lines.append(f"- {item.get('branch', 'unknown')}: {item.get('error', 'unknown error')}")
|
||||
|
||||
if mixed_results is not None:
|
||||
lines.append("")
|
||||
lines.append(f"🧩 混合结果（{len(mixed_results)} 条）：")
|
||||
for idx, item in enumerate(mixed_results[:5], start=1):
|
||||
src = ",".join(item.get("source_branches", []) or [])
|
||||
if str(item.get("type", "") or "") == "episode":
|
||||
title = str(item.get("title", "") or "Untitled")
|
||||
lines.append(f"{idx}. 🧠 {title} [{src}]")
|
||||
else:
|
||||
text = str(item.get("content", "") or "")
|
||||
if len(text) > 80:
|
||||
text = text[:80] + "..."
|
||||
lines.append(f"{idx}. {text} [{src}]")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
top_k: int,
|
||||
mix: bool,
|
||||
mix_top_k: Optional[int],
|
||||
time_from: Optional[str],
|
||||
time_to: Optional[str],
|
||||
search_runner: Optional[BranchRunner],
|
||||
time_runner: Optional[BranchRunner],
|
||||
episode_runner: Optional[BranchRunner],
|
||||
) -> Dict[str, Any]:
|
||||
clean_query = str(query or "").strip()
|
||||
safe_top_k = max(1, int(top_k))
|
||||
safe_mix_top_k = max(1, int(mix_top_k if mix_top_k is not None else safe_top_k))
|
||||
|
||||
branches: Dict[str, Dict[str, Any]] = {}
|
||||
errors: List[Dict[str, str]] = []
|
||||
scheduled: List[Tuple[str, asyncio.Task]] = []
|
||||
|
||||
if clean_query:
|
||||
if search_runner is not None:
|
||||
scheduled.append(("search", asyncio.create_task(search_runner())))
|
||||
else:
|
||||
branches["search"] = self._normalize_branch_payload(
|
||||
"search",
|
||||
{"success": False, "error": "search runner unavailable", "results": []},
|
||||
)
|
||||
else:
|
||||
branches["search"] = self._normalize_branch_payload(
|
||||
"search",
|
||||
{
|
||||
"success": False,
|
||||
"skipped": True,
|
||||
"skip_reason": "missing_query",
|
||||
"results": [],
|
||||
"count": 0,
|
||||
},
|
||||
)
|
||||
|
||||
if time_from or time_to:
|
||||
if time_runner is not None:
|
||||
scheduled.append(("time", asyncio.create_task(time_runner())))
|
||||
else:
|
||||
branches["time"] = self._normalize_branch_payload(
|
||||
"time",
|
||||
{"success": False, "error": "time runner unavailable", "results": []},
|
||||
)
|
||||
else:
|
||||
branches["time"] = self._normalize_branch_payload(
|
||||
"time",
|
||||
{
|
||||
"success": False,
|
||||
"skipped": True,
|
||||
"skip_reason": "missing_time_window",
|
||||
"results": [],
|
||||
"count": 0,
|
||||
},
|
||||
)
|
||||
|
||||
if episode_runner is not None:
|
||||
scheduled.append(("episode", asyncio.create_task(episode_runner())))
|
||||
else:
|
||||
branches["episode"] = self._normalize_branch_payload(
|
||||
"episode",
|
||||
{"success": False, "error": "episode runner unavailable", "results": []},
|
||||
)
|
||||
|
||||
if scheduled:
|
||||
done = await asyncio.gather(
|
||||
*[task for _, task in scheduled],
|
||||
return_exceptions=True,
|
||||
)
|
||||
for (branch_name, _), payload in zip(scheduled, done):
|
||||
if isinstance(payload, Exception):
|
||||
logger.error(f"aggregate branch failed: branch={branch_name} error={payload}")
|
||||
normalized = self._normalize_branch_payload(
|
||||
branch_name,
|
||||
{
|
||||
"success": False,
|
||||
"error": str(payload),
|
||||
"results": [],
|
||||
},
|
||||
)
|
||||
else:
|
||||
normalized = self._normalize_branch_payload(branch_name, payload)
|
||||
branches[branch_name] = normalized
|
||||
|
||||
for name in ("search", "time", "episode"):
|
||||
branch = branches.get(name)
|
||||
if not branch:
|
||||
continue
|
||||
if branch.get("skipped", False):
|
||||
continue
|
||||
if not branch.get("success", False):
|
||||
errors.append(
|
||||
{
|
||||
"branch": name,
|
||||
"error": str(branch.get("error", "") or "unknown error"),
|
||||
}
|
||||
)
|
||||
|
||||
success = any(
|
||||
bool(branches.get(name, {}).get("success", False))
|
||||
for name in ("search", "time", "episode")
|
||||
)
|
||||
mixed_results: Optional[List[Dict[str, Any]]] = None
|
||||
if mix:
|
||||
mixed_results = self._build_mixed_results(branches=branches, top_k=safe_mix_top_k)
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"success": success,
|
||||
"query_type": "aggregate",
|
||||
"query": clean_query,
|
||||
"top_k": safe_top_k,
|
||||
"mix": bool(mix),
|
||||
"mix_top_k": safe_mix_top_k,
|
||||
"branches": branches,
|
||||
"errors": errors,
|
||||
"summary": self._build_summary(branches),
|
||||
}
|
||||
if mixed_results is not None:
|
||||
payload["mixed_results"] = mixed_results
|
||||
|
||||
payload["content"] = self._build_content(
|
||||
query=clean_query,
|
||||
branches=branches,
|
||||
errors=errors,
|
||||
mixed_results=mixed_results,
|
||||
)
|
||||
return payload
|
||||
182
src/A_memorix/core/utils/episode_retrieval_service.py
Normal file
182
src/A_memorix/core/utils/episode_retrieval_service.py
Normal file
@@ -0,0 +1,182 @@
|
||||
"""Episode hybrid retrieval service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from src.common.logger import get_logger
|
||||
|
||||
from ..retrieval import DualPathRetriever, TemporalQueryOptions
|
||||
|
||||
logger = get_logger("A_Memorix.EpisodeRetrievalService")
|
||||
|
||||
|
||||
class EpisodeRetrievalService:
|
||||
"""Hybrid episode retrieval backed by lexical rows and evidence projection."""
|
||||
|
||||
_RRF_K = 60.0
|
||||
_BRANCH_WEIGHTS = {
|
||||
"lexical": 1.0,
|
||||
"paragraph_evidence": 1.0,
|
||||
"relation_evidence": 0.85,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
metadata_store: Any,
|
||||
retriever: Optional[DualPathRetriever] = None,
|
||||
) -> None:
|
||||
self.metadata_store = metadata_store
|
||||
self.retriever = retriever
|
||||
|
||||
async def query(
|
||||
self,
|
||||
*,
|
||||
query: str = "",
|
||||
top_k: int = 5,
|
||||
time_from: Optional[float] = None,
|
||||
time_to: Optional[float] = None,
|
||||
person: Optional[str] = None,
|
||||
source: Optional[str] = None,
|
||||
include_paragraphs: bool = False,
|
||||
) -> List[Dict[str, Any]]:
|
||||
clean_query = str(query or "").strip()
|
||||
safe_top_k = max(1, int(top_k))
|
||||
candidate_k = max(30, safe_top_k * 6)
|
||||
|
||||
branches: Dict[str, List[Dict[str, Any]]] = {
|
||||
"lexical": self.metadata_store.query_episodes(
|
||||
query=clean_query,
|
||||
time_from=time_from,
|
||||
time_to=time_to,
|
||||
person=person,
|
||||
source=source,
|
||||
limit=(candidate_k if clean_query else safe_top_k),
|
||||
)
|
||||
}
|
||||
|
||||
if clean_query and self.retriever is not None:
|
||||
try:
|
||||
temporal = TemporalQueryOptions(
|
||||
time_from=time_from,
|
||||
time_to=time_to,
|
||||
person=person,
|
||||
source=source,
|
||||
)
|
||||
results = await self.retriever.retrieve(
|
||||
query=clean_query,
|
||||
top_k=candidate_k,
|
||||
temporal=temporal,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(f"episode evidence retrieval failed, fallback to lexical only: {exc}")
|
||||
else:
|
||||
paragraph_rank_map: Dict[str, int] = {}
|
||||
relation_rank_map: Dict[str, int] = {}
|
||||
for rank, item in enumerate(results, start=1):
|
||||
hash_value = str(getattr(item, "hash_value", "") or "").strip()
|
||||
result_type = str(getattr(item, "result_type", "") or "").strip().lower()
|
||||
if not hash_value:
|
||||
continue
|
||||
if result_type == "paragraph" and hash_value not in paragraph_rank_map:
|
||||
paragraph_rank_map[hash_value] = rank
|
||||
elif result_type == "relation" and hash_value not in relation_rank_map:
|
||||
relation_rank_map[hash_value] = rank
|
||||
|
||||
if paragraph_rank_map:
|
||||
paragraph_rows = self.metadata_store.get_episode_rows_by_paragraph_hashes(
|
||||
list(paragraph_rank_map.keys()),
|
||||
source=source,
|
||||
)
|
||||
if paragraph_rows:
|
||||
branches["paragraph_evidence"] = self._rank_projected_rows(
|
||||
paragraph_rows,
|
||||
rank_map=paragraph_rank_map,
|
||||
support_key="matched_paragraph_hashes",
|
||||
)
|
||||
|
||||
if relation_rank_map:
|
||||
relation_rows = self.metadata_store.get_episode_rows_by_relation_hashes(
|
||||
list(relation_rank_map.keys()),
|
||||
source=source,
|
||||
)
|
||||
if relation_rows:
|
||||
branches["relation_evidence"] = self._rank_projected_rows(
|
||||
relation_rows,
|
||||
rank_map=relation_rank_map,
|
||||
support_key="matched_relation_hashes",
|
||||
)
|
||||
|
||||
fused = self._fuse_branches(branches, top_k=safe_top_k)
|
||||
if include_paragraphs:
|
||||
for item in fused:
|
||||
item["paragraphs"] = self.metadata_store.get_episode_paragraphs(
|
||||
episode_id=str(item.get("episode_id") or ""),
|
||||
limit=50,
|
||||
)
|
||||
return fused
|
||||
|
||||
@staticmethod
|
||||
def _rank_projected_rows(
|
||||
rows: List[Dict[str, Any]],
|
||||
*,
|
||||
rank_map: Dict[str, int],
|
||||
support_key: str,
|
||||
) -> List[Dict[str, Any]]:
|
||||
sentinel = 10**9
|
||||
ranked = [dict(item) for item in rows]
|
||||
|
||||
def _first_support_rank(item: Dict[str, Any]) -> int:
|
||||
support_hashes = [str(x or "").strip() for x in (item.get(support_key) or [])]
|
||||
ranks = [int(rank_map[h]) for h in support_hashes if h in rank_map]
|
||||
return min(ranks) if ranks else sentinel
|
||||
|
||||
ranked.sort(
|
||||
key=lambda item: (
|
||||
_first_support_rank(item),
|
||||
-int(item.get("matched_paragraph_count") or 0),
|
||||
-float(item.get("updated_at") or 0.0),
|
||||
str(item.get("episode_id") or ""),
|
||||
)
|
||||
)
|
||||
return ranked
|
||||
|
||||
def _fuse_branches(
|
||||
self,
|
||||
branches: Dict[str, List[Dict[str, Any]]],
|
||||
*,
|
||||
top_k: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
bucket: Dict[str, Dict[str, Any]] = {}
|
||||
for branch_name, rows in branches.items():
|
||||
weight = float(self._BRANCH_WEIGHTS.get(branch_name, 0.0) or 0.0)
|
||||
if weight <= 0.0:
|
||||
continue
|
||||
for rank, row in enumerate(rows, start=1):
|
||||
episode_id = str(row.get("episode_id", "") or "").strip()
|
||||
if not episode_id:
|
||||
continue
|
||||
if episode_id not in bucket:
|
||||
payload = dict(row)
|
||||
payload.pop("matched_paragraph_hashes", None)
|
||||
payload.pop("matched_relation_hashes", None)
|
||||
payload.pop("matched_paragraph_count", None)
|
||||
payload.pop("matched_relation_count", None)
|
||||
payload["_fusion_score"] = 0.0
|
||||
bucket[episode_id] = payload
|
||||
bucket[episode_id]["_fusion_score"] = float(
|
||||
bucket[episode_id].get("_fusion_score", 0.0)
|
||||
) + weight / (self._RRF_K + float(rank))
|
||||
|
||||
out = list(bucket.values())
|
||||
out.sort(
|
||||
key=lambda item: (
|
||||
-float(item.get("_fusion_score", 0.0)),
|
||||
-float(item.get("updated_at") or 0.0),
|
||||
str(item.get("episode_id") or ""),
|
||||
)
|
||||
)
|
||||
for item in out:
|
||||
item.pop("_fusion_score", None)
|
||||
return out[: max(1, int(top_k))]
|
||||
304
src/A_memorix/core/utils/episode_segmentation_service.py
Normal file
304
src/A_memorix/core/utils/episode_segmentation_service.py
Normal file
@@ -0,0 +1,304 @@
|
||||
"""
|
||||
Episode 语义切分服务（LLM 主路径）。
|
||||
|
||||
职责：
|
||||
1. 组装语义切分提示词
|
||||
2. 调用 LLM 生成结构化 episode JSON
|
||||
3. 严格校验输出结构，返回标准化结果
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from src.common.logger import get_logger
|
||||
from src.config.model_configs import TaskConfig
|
||||
from src.config.config import model_config as host_model_config
|
||||
from src.services import llm_service as llm_api
|
||||
|
||||
logger = get_logger("A_Memorix.EpisodeSegmentationService")
|
||||
|
||||
|
||||
class EpisodeSegmentationService:
|
||||
"""基于 LLM 的 episode 语义切分服务。"""
|
||||
|
||||
SEGMENTATION_VERSION = "episode_mvp_v1"
|
||||
|
||||
def __init__(self, plugin_config: Optional[dict] = None):
|
||||
self.plugin_config = plugin_config or {}
|
||||
|
||||
def _cfg(self, key: str, default: Any = None) -> Any:
|
||||
current: Any = self.plugin_config
|
||||
for part in key.split("."):
|
||||
if isinstance(current, dict) and part in current:
|
||||
current = current[part]
|
||||
else:
|
||||
return default
|
||||
return current
|
||||
|
||||
@staticmethod
|
||||
def _is_task_config(obj: Any) -> bool:
|
||||
return hasattr(obj, "model_list") and bool(getattr(obj, "model_list", []))
|
||||
|
||||
def _build_single_model_task(self, model_name: str, template: TaskConfig) -> TaskConfig:
|
||||
return TaskConfig(
|
||||
model_list=[model_name],
|
||||
max_tokens=template.max_tokens,
|
||||
temperature=template.temperature,
|
||||
slow_threshold=template.slow_threshold,
|
||||
selection_strategy=template.selection_strategy,
|
||||
)
|
||||
|
||||
def _pick_template_task(self, available_tasks: Dict[str, Any]) -> Optional[TaskConfig]:
|
||||
preferred = ("utils", "replyer", "planner", "tool_use")
|
||||
for task_name in preferred:
|
||||
cfg = available_tasks.get(task_name)
|
||||
if self._is_task_config(cfg):
|
||||
return cfg
|
||||
for task_name, cfg in available_tasks.items():
|
||||
if task_name != "embedding" and self._is_task_config(cfg):
|
||||
return cfg
|
||||
for cfg in available_tasks.values():
|
||||
if self._is_task_config(cfg):
|
||||
return cfg
|
||||
return None
|
||||
|
||||
def _resolve_model_config(self) -> Tuple[Optional[Any], str]:
|
||||
available_tasks = llm_api.get_available_models() or {}
|
||||
if not available_tasks:
|
||||
return None, "unavailable"
|
||||
|
||||
selector = str(self._cfg("episode.segmentation_model", "auto") or "auto").strip()
|
||||
model_dict = getattr(host_model_config, "models_dict", {}) or {}
|
||||
|
||||
if selector and selector.lower() != "auto":
|
||||
direct_task = available_tasks.get(selector)
|
||||
if self._is_task_config(direct_task):
|
||||
return direct_task, selector
|
||||
|
||||
if selector in model_dict:
|
||||
template = self._pick_template_task(available_tasks)
|
||||
if template is not None:
|
||||
return self._build_single_model_task(selector, template), selector
|
||||
|
||||
logger.warning(f"episode.segmentation_model='{selector}' 不可用，回退 auto")
|
||||
|
||||
for task_name in ("utils", "replyer", "planner", "tool_use"):
|
||||
cfg = available_tasks.get(task_name)
|
||||
if self._is_task_config(cfg):
|
||||
return cfg, task_name
|
||||
|
||||
fallback = self._pick_template_task(available_tasks)
|
||||
if fallback is not None:
|
||||
return fallback, "auto"
|
||||
return None, "unavailable"
|
||||
|
||||
@staticmethod
|
||||
def _clamp_score(value: Any, default: float = 0.0) -> float:
|
||||
try:
|
||||
num = float(value)
|
||||
except Exception:
|
||||
num = default
|
||||
if num < 0.0:
|
||||
return 0.0
|
||||
if num > 1.0:
|
||||
return 1.0
|
||||
return num
|
||||
|
||||
@staticmethod
|
||||
def _safe_json_loads(text: str) -> Dict[str, Any]:
|
||||
raw = str(text or "").strip()
|
||||
if not raw:
|
||||
raise ValueError("empty_response")
|
||||
|
||||
if "```" in raw:
|
||||
raw = raw.replace("```json", "```").replace("```JSON", "```")
|
||||
parts = raw.split("```")
|
||||
for part in parts:
|
||||
part = part.strip()
|
||||
if part.startswith("{") and part.endswith("}"):
|
||||
raw = part
|
||||
break
|
||||
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict):
|
||||
return data
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
start = raw.find("{")
|
||||
end = raw.rfind("}")
|
||||
if start >= 0 and end > start:
|
||||
candidate = raw[start : end + 1]
|
||||
data = json.loads(candidate)
|
||||
if isinstance(data, dict):
|
||||
return data
|
||||
|
||||
raise ValueError("invalid_json_response")
|
||||
|
||||
def _build_prompt(
|
||||
self,
|
||||
*,
|
||||
source: str,
|
||||
window_start: Optional[float],
|
||||
window_end: Optional[float],
|
||||
paragraphs: List[Dict[str, Any]],
|
||||
) -> str:
|
||||
rows: List[str] = []
|
||||
for idx, item in enumerate(paragraphs, 1):
|
||||
p_hash = str(item.get("hash", "") or "").strip()
|
||||
content = str(item.get("content", "") or "").strip().replace("\r\n", "\n")
|
||||
content = content[:800]
|
||||
event_start = item.get("event_time_start")
|
||||
event_end = item.get("event_time_end")
|
||||
event_time = item.get("event_time")
|
||||
rows.append(
|
||||
(
|
||||
f"[{idx}] hash={p_hash}\n"
|
||||
f"event_time={event_time}\n"
|
||||
f"event_time_start={event_start}\n"
|
||||
f"event_time_end={event_end}\n"
|
||||
f"content={content}"
|
||||
)
|
||||
)
|
||||
|
||||
source_text = str(source or "").strip() or "unknown"
|
||||
return (
|
||||
"You are an episode segmentation engine.\n"
|
||||
"Group the given paragraphs into one or more coherent episodes.\n"
|
||||
"Return JSON ONLY. No markdown, no explanation.\n"
|
||||
"\n"
|
||||
"Hard JSON schema:\n"
|
||||
"{\n"
|
||||
' "episodes": [\n'
|
||||
" {\n"
|
||||
' "title": "string",\n'
|
||||
' "summary": "string",\n'
|
||||
' "paragraph_hashes": ["hash1", "hash2"],\n'
|
||||
' "participants": ["person1", "person2"],\n'
|
||||
' "keywords": ["kw1", "kw2"],\n'
|
||||
' "time_confidence": 0.0,\n'
|
||||
' "llm_confidence": 0.0\n'
|
||||
" }\n"
|
||||
" ]\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"Rules:\n"
|
||||
"1) paragraph_hashes must come from input only.\n"
|
||||
"2) title and summary must be non-empty.\n"
|
||||
"3) keep participants/keywords concise and deduplicated.\n"
|
||||
"4) if uncertain, still provide best effort confidence values.\n"
|
||||
"\n"
|
||||
f"source={source_text}\n"
|
||||
f"window_start={window_start}\n"
|
||||
f"window_end={window_end}\n"
|
||||
"paragraphs:\n"
|
||||
+ "\n\n".join(rows)
|
||||
)
|
||||
|
||||
def _normalize_episodes(
|
||||
self,
|
||||
*,
|
||||
payload: Dict[str, Any],
|
||||
input_hashes: List[str],
|
||||
) -> List[Dict[str, Any]]:
|
||||
raw_episodes = payload.get("episodes")
|
||||
if not isinstance(raw_episodes, list):
|
||||
raise ValueError("episodes_missing_or_not_list")
|
||||
|
||||
valid_hashes = set(input_hashes)
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
for item in raw_episodes:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
title = str(item.get("title", "") or "").strip()
|
||||
summary = str(item.get("summary", "") or "").strip()
|
||||
if not title or not summary:
|
||||
continue
|
||||
|
||||
raw_hashes = item.get("paragraph_hashes")
|
||||
if not isinstance(raw_hashes, list):
|
||||
continue
|
||||
|
||||
dedup_hashes: List[str] = []
|
||||
seen_hashes = set()
|
||||
for h in raw_hashes:
|
||||
token = str(h or "").strip()
|
||||
if not token or token in seen_hashes or token not in valid_hashes:
|
||||
continue
|
||||
seen_hashes.add(token)
|
||||
dedup_hashes.append(token)
|
||||
|
||||
if not dedup_hashes:
|
||||
continue
|
||||
|
||||
participants = []
|
||||
for p in item.get("participants", []) or []:
|
||||
token = str(p or "").strip()
|
||||
if token:
|
||||
participants.append(token)
|
||||
|
||||
keywords = []
|
||||
for kw in item.get("keywords", []) or []:
|
||||
token = str(kw or "").strip()
|
||||
if token:
|
||||
keywords.append(token)
|
||||
|
||||
normalized.append(
|
||||
{
|
||||
"title": title,
|
||||
"summary": summary,
|
||||
"paragraph_hashes": dedup_hashes,
|
||||
"participants": participants[:16],
|
||||
"keywords": keywords[:20],
|
||||
"time_confidence": self._clamp_score(item.get("time_confidence"), default=1.0),
|
||||
"llm_confidence": self._clamp_score(item.get("llm_confidence"), default=0.5),
|
||||
}
|
||||
)
|
||||
|
||||
if not normalized:
|
||||
raise ValueError("episodes_all_invalid")
|
||||
return normalized
|
||||
|
||||
async def segment(
|
||||
self,
|
||||
*,
|
||||
source: str,
|
||||
window_start: Optional[float],
|
||||
window_end: Optional[float],
|
||||
paragraphs: List[Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
if not paragraphs:
|
||||
raise ValueError("paragraphs_empty")
|
||||
|
||||
model_config, model_label = self._resolve_model_config()
|
||||
if model_config is None:
|
||||
raise RuntimeError("episode segmentation model unavailable")
|
||||
|
||||
prompt = self._build_prompt(
|
||||
source=source,
|
||||
window_start=window_start,
|
||||
window_end=window_end,
|
||||
paragraphs=paragraphs,
|
||||
)
|
||||
success, response, _, _ = await llm_api.generate_with_model(
|
||||
prompt=prompt,
|
||||
model_config=model_config,
|
||||
request_type="A_Memorix.EpisodeSegmentation",
|
||||
)
|
||||
if not success or not response:
|
||||
raise RuntimeError("llm_generate_failed")
|
||||
|
||||
payload = self._safe_json_loads(str(response))
|
||||
input_hashes = [str(p.get("hash", "") or "").strip() for p in paragraphs]
|
||||
episodes = self._normalize_episodes(payload=payload, input_hashes=input_hashes)
|
||||
|
||||
return {
|
||||
"episodes": episodes,
|
||||
"segmentation_model": model_label,
|
||||
"segmentation_version": self.SEGMENTATION_VERSION,
|
||||
}
|
||||
|
||||
558
src/A_memorix/core/utils/episode_service.py
Normal file
558
src/A_memorix/core/utils/episode_service.py
Normal file
@@ -0,0 +1,558 @@
|
||||
"""
|
||||
Episode 聚合与落库服务。
|
||||
|
||||
流程：
|
||||
1. 从 pending 队列读取段落并组批
|
||||
2. 按 source + 时间窗口切组
|
||||
3. 调用 LLM 语义切分
|
||||
4. 写入 episodes + episode_paragraphs
|
||||
5. LLM 失败时使用确定性 fallback
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from src.common.logger import get_logger
|
||||
|
||||
from .episode_segmentation_service import EpisodeSegmentationService
|
||||
from .hash import compute_hash
|
||||
|
||||
logger = get_logger("A_Memorix.EpisodeService")
|
||||
|
||||
|
||||
class EpisodeService:
|
||||
"""Episode MVP 后台处理服务。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
metadata_store: Any,
|
||||
plugin_config: Optional[Any] = None,
|
||||
segmentation_service: Optional[EpisodeSegmentationService] = None,
|
||||
):
|
||||
self.metadata_store = metadata_store
|
||||
self.plugin_config = plugin_config or {}
|
||||
self.segmentation_service = segmentation_service or EpisodeSegmentationService(
|
||||
plugin_config=self._config_dict(),
|
||||
)
|
||||
|
||||
def _config_dict(self) -> Dict[str, Any]:
|
||||
if isinstance(self.plugin_config, dict):
|
||||
return self.plugin_config
|
||||
return {}
|
||||
|
||||
def _cfg(self, key: str, default: Any = None) -> Any:
|
||||
getter = getattr(self.plugin_config, "get_config", None)
|
||||
if callable(getter):
|
||||
return getter(key, default)
|
||||
|
||||
current: Any = self.plugin_config
|
||||
for part in key.split("."):
|
||||
if isinstance(current, dict) and part in current:
|
||||
current = current[part]
|
||||
else:
|
||||
return default
|
||||
return current
|
||||
|
||||
@staticmethod
|
||||
def _to_optional_float(value: Any) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _clamp_score(value: Any, default: float = 1.0) -> float:
|
||||
try:
|
||||
num = float(value)
|
||||
except Exception:
|
||||
num = default
|
||||
if num < 0.0:
|
||||
return 0.0
|
||||
if num > 1.0:
|
||||
return 1.0
|
||||
return num
|
||||
|
||||
@staticmethod
|
||||
def _paragraph_anchor(paragraph: Dict[str, Any]) -> float:
|
||||
for key in ("event_time_end", "event_time_start", "event_time", "created_at"):
|
||||
value = paragraph.get(key)
|
||||
try:
|
||||
if value is not None:
|
||||
return float(value)
|
||||
except Exception:
|
||||
continue
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _paragraph_sort_key(paragraph: Dict[str, Any]) -> Tuple[float, str]:
|
||||
return (
|
||||
EpisodeService._paragraph_anchor(paragraph),
|
||||
str(paragraph.get("hash", "") or ""),
|
||||
)
|
||||
|
||||
def load_pending_paragraphs(
|
||||
self,
|
||||
pending_rows: List[Dict[str, Any]],
|
||||
) -> Tuple[List[Dict[str, Any]], List[str]]:
|
||||
"""
|
||||
将 pending 行展开为段落上下文。
|
||||
|
||||
Returns:
|
||||
(loaded_paragraphs, missing_hashes)
|
||||
"""
|
||||
loaded: List[Dict[str, Any]] = []
|
||||
missing: List[str] = []
|
||||
for row in pending_rows or []:
|
||||
p_hash = str(row.get("paragraph_hash", "") or "").strip()
|
||||
if not p_hash:
|
||||
continue
|
||||
|
||||
paragraph = self.metadata_store.get_paragraph(p_hash)
|
||||
if not paragraph:
|
||||
missing.append(p_hash)
|
||||
continue
|
||||
|
||||
loaded.append(
|
||||
{
|
||||
"hash": p_hash,
|
||||
"source": str(row.get("source") or paragraph.get("source") or "").strip(),
|
||||
"content": str(paragraph.get("content", "") or ""),
|
||||
"created_at": self._to_optional_float(paragraph.get("created_at"))
|
||||
or self._to_optional_float(row.get("created_at"))
|
||||
or 0.0,
|
||||
"event_time": self._to_optional_float(paragraph.get("event_time")),
|
||||
"event_time_start": self._to_optional_float(paragraph.get("event_time_start")),
|
||||
"event_time_end": self._to_optional_float(paragraph.get("event_time_end")),
|
||||
"time_granularity": str(paragraph.get("time_granularity", "") or "").strip() or None,
|
||||
"time_confidence": self._clamp_score(paragraph.get("time_confidence"), default=1.0),
|
||||
}
|
||||
)
|
||||
return loaded, missing
|
||||
|
||||
def group_paragraphs(self, paragraphs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
按 source + 时间邻近窗口组批，并受段落数/字符数上限约束。
|
||||
"""
|
||||
if not paragraphs:
|
||||
return []
|
||||
|
||||
max_paragraphs = max(1, int(self._cfg("episode.max_paragraphs_per_call", 20)))
|
||||
max_chars = max(200, int(self._cfg("episode.max_chars_per_call", 6000)))
|
||||
window_seconds = max(
|
||||
60.0,
|
||||
float(self._cfg("episode.source_time_window_hours", 24)) * 3600.0,
|
||||
)
|
||||
|
||||
by_source: Dict[str, List[Dict[str, Any]]] = {}
|
||||
for paragraph in paragraphs:
|
||||
source = str(paragraph.get("source", "") or "").strip()
|
||||
by_source.setdefault(source, []).append(paragraph)
|
||||
|
||||
groups: List[Dict[str, Any]] = []
|
||||
for source, items in by_source.items():
|
||||
ordered = sorted(items, key=self._paragraph_sort_key)
|
||||
|
||||
current: List[Dict[str, Any]] = []
|
||||
current_chars = 0
|
||||
last_anchor: Optional[float] = None
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal current, current_chars, last_anchor
|
||||
if not current:
|
||||
return
|
||||
sorted_current = sorted(current, key=self._paragraph_sort_key)
|
||||
groups.append(
|
||||
{
|
||||
"source": source,
|
||||
"paragraphs": sorted_current,
|
||||
}
|
||||
)
|
||||
current = []
|
||||
current_chars = 0
|
||||
last_anchor = None
|
||||
|
||||
for paragraph in ordered:
|
||||
anchor = self._paragraph_anchor(paragraph)
|
||||
content_len = len(str(paragraph.get("content", "") or ""))
|
||||
|
||||
need_flush = False
|
||||
if current:
|
||||
if len(current) >= max_paragraphs:
|
||||
need_flush = True
|
||||
elif current_chars + content_len > max_chars:
|
||||
need_flush = True
|
||||
elif last_anchor is not None and abs(anchor - last_anchor) > window_seconds:
|
||||
need_flush = True
|
||||
|
||||
if need_flush:
|
||||
flush()
|
||||
|
||||
current.append(paragraph)
|
||||
current_chars += content_len
|
||||
last_anchor = anchor
|
||||
|
||||
flush()
|
||||
|
||||
groups.sort(
|
||||
key=lambda g: self._paragraph_anchor(g["paragraphs"][0]) if g.get("paragraphs") else 0.0
|
||||
)
|
||||
return groups
|
||||
|
||||
def _compute_time_meta(self, paragraphs: List[Dict[str, Any]]) -> Tuple[Optional[float], Optional[float], Optional[str], float]:
|
||||
starts: List[float] = []
|
||||
ends: List[float] = []
|
||||
granularity_priority = {
|
||||
"minute": 4,
|
||||
"hour": 3,
|
||||
"day": 2,
|
||||
"month": 1,
|
||||
"year": 0,
|
||||
}
|
||||
granularity = None
|
||||
granularity_rank = -1
|
||||
conf_values: List[float] = []
|
||||
|
||||
for p in paragraphs:
|
||||
s = self._to_optional_float(p.get("event_time_start"))
|
||||
e = self._to_optional_float(p.get("event_time_end"))
|
||||
t = self._to_optional_float(p.get("event_time"))
|
||||
c = self._to_optional_float(p.get("created_at"))
|
||||
|
||||
start_candidate = s if s is not None else (t if t is not None else (e if e is not None else c))
|
||||
end_candidate = e if e is not None else (t if t is not None else (s if s is not None else c))
|
||||
|
||||
if start_candidate is not None:
|
||||
starts.append(start_candidate)
|
||||
if end_candidate is not None:
|
||||
ends.append(end_candidate)
|
||||
|
||||
g = str(p.get("time_granularity", "") or "").strip().lower()
|
||||
if g in granularity_priority and granularity_priority[g] > granularity_rank:
|
||||
granularity_rank = granularity_priority[g]
|
||||
granularity = g
|
||||
|
||||
conf_values.append(self._clamp_score(p.get("time_confidence"), default=1.0))
|
||||
|
||||
time_start = min(starts) if starts else None
|
||||
time_end = max(ends) if ends else None
|
||||
time_conf = sum(conf_values) / len(conf_values) if conf_values else 1.0
|
||||
return time_start, time_end, granularity, self._clamp_score(time_conf, default=1.0)
|
||||
|
||||
def _collect_participants(self, paragraph_hashes: List[str], limit: int = 16) -> List[str]:
|
||||
seen = set()
|
||||
participants: List[str] = []
|
||||
for p_hash in paragraph_hashes:
|
||||
try:
|
||||
entities = self.metadata_store.get_paragraph_entities(p_hash)
|
||||
except Exception:
|
||||
entities = []
|
||||
for item in entities:
|
||||
name = str(item.get("name", "") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
key = name.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
participants.append(name)
|
||||
if len(participants) >= limit:
|
||||
return participants
|
||||
return participants
|
||||
|
||||
@staticmethod
|
||||
def _derive_keywords(paragraphs: List[Dict[str, Any]], limit: int = 12) -> List[str]:
|
||||
token_counter: Counter[str] = Counter()
|
||||
token_pattern = re.compile(r"[A-Za-z0-9_\u4e00-\u9fff]{2,}")
|
||||
stop_words = {
|
||||
"the",
|
||||
"and",
|
||||
"that",
|
||||
"this",
|
||||
"with",
|
||||
"from",
|
||||
"for",
|
||||
"have",
|
||||
"will",
|
||||
"your",
|
||||
"you",
|
||||
"我们",
|
||||
"你们",
|
||||
"他们",
|
||||
"以及",
|
||||
"一个",
|
||||
"这个",
|
||||
"那个",
|
||||
"然后",
|
||||
"因为",
|
||||
"所以",
|
||||
}
|
||||
for p in paragraphs:
|
||||
text = str(p.get("content", "") or "").lower()
|
||||
for token in token_pattern.findall(text):
|
||||
if token in stop_words:
|
||||
continue
|
||||
token_counter[token] += 1
|
||||
|
||||
return [token for token, _ in token_counter.most_common(limit)]
|
||||
|
||||
def _build_fallback_episode(self, group: Dict[str, Any]) -> Dict[str, Any]:
|
||||
paragraphs = group.get("paragraphs", []) or []
|
||||
source = str(group.get("source", "") or "").strip()
|
||||
hashes = [str(p.get("hash", "") or "").strip() for p in paragraphs if str(p.get("hash", "") or "").strip()]
|
||||
snippets = []
|
||||
for p in paragraphs[:3]:
|
||||
text = str(p.get("content", "") or "").strip().replace("\n", " ")
|
||||
if text:
|
||||
snippets.append(text[:140])
|
||||
summary = "；".join(snippets)[:500] if snippets else "自动回退生成的情景记忆。"
|
||||
|
||||
time_start, time_end, granularity, time_conf = self._compute_time_meta(paragraphs)
|
||||
participants = self._collect_participants(hashes, limit=12)
|
||||
keywords = self._derive_keywords(paragraphs, limit=10)
|
||||
|
||||
if time_start is not None:
|
||||
day_text = datetime.fromtimestamp(time_start).strftime("%Y-%m-%d")
|
||||
title = f"{source or 'unknown'} {day_text} 情景片段"
|
||||
else:
|
||||
title = f"{source or 'unknown'} 情景片段"
|
||||
|
||||
return {
|
||||
"title": title[:80],
|
||||
"summary": summary,
|
||||
"paragraph_hashes": hashes,
|
||||
"participants": participants,
|
||||
"keywords": keywords,
|
||||
"time_confidence": time_conf,
|
||||
"llm_confidence": 0.0,
|
||||
"event_time_start": time_start,
|
||||
"event_time_end": time_end,
|
||||
"time_granularity": granularity,
|
||||
"segmentation_model": "fallback_rule",
|
||||
"segmentation_version": EpisodeSegmentationService.SEGMENTATION_VERSION,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_episode_hashes(episode_hashes: List[str], group_hashes_ordered: List[str]) -> List[str]:
|
||||
in_group = set(group_hashes_ordered)
|
||||
dedup: List[str] = []
|
||||
seen = set()
|
||||
for h in episode_hashes or []:
|
||||
token = str(h or "").strip()
|
||||
if not token or token not in in_group or token in seen:
|
||||
continue
|
||||
seen.add(token)
|
||||
dedup.append(token)
|
||||
return dedup
|
||||
|
||||
async def _build_episode_payloads_for_group(self, group: Dict[str, Any]) -> Dict[str, Any]:
|
||||
paragraphs = group.get("paragraphs", []) or []
|
||||
if not paragraphs:
|
||||
return {
|
||||
"payloads": [],
|
||||
"done_hashes": [],
|
||||
"episode_count": 0,
|
||||
"fallback_count": 0,
|
||||
}
|
||||
|
||||
source = str(group.get("source", "") or "").strip()
|
||||
group_hashes = [str(p.get("hash", "") or "").strip() for p in paragraphs if str(p.get("hash", "") or "").strip()]
|
||||
group_start, group_end, _, _ = self._compute_time_meta(paragraphs)
|
||||
|
||||
fallback_used = False
|
||||
segmentation_model = "fallback_rule"
|
||||
segmentation_version = EpisodeSegmentationService.SEGMENTATION_VERSION
|
||||
|
||||
try:
|
||||
llm_result = await self.segmentation_service.segment(
|
||||
source=source,
|
||||
window_start=group_start,
|
||||
window_end=group_end,
|
||||
paragraphs=paragraphs,
|
||||
)
|
||||
episodes = list(llm_result.get("episodes") or [])
|
||||
segmentation_model = str(llm_result.get("segmentation_model", "") or "").strip() or "auto"
|
||||
segmentation_version = str(llm_result.get("segmentation_version", "") or "").strip() or EpisodeSegmentationService.SEGMENTATION_VERSION
|
||||
if not episodes:
|
||||
raise ValueError("llm_empty_episodes")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Episode segmentation fallback: "
|
||||
f"source={source} "
|
||||
f"size={len(group_hashes)} "
|
||||
f"err={e}"
|
||||
)
|
||||
episodes = [self._build_fallback_episode(group)]
|
||||
fallback_used = True
|
||||
|
||||
stored_payloads: List[Dict[str, Any]] = []
|
||||
for episode in episodes:
|
||||
ordered_hashes = self._normalize_episode_hashes(
|
||||
episode_hashes=episode.get("paragraph_hashes", []),
|
||||
group_hashes_ordered=group_hashes,
|
||||
)
|
||||
if not ordered_hashes:
|
||||
continue
|
||||
|
||||
sub_paragraphs = [p for p in paragraphs if str(p.get("hash", "") or "") in set(ordered_hashes)]
|
||||
event_start, event_end, granularity, time_conf_default = self._compute_time_meta(sub_paragraphs)
|
||||
|
||||
participants = [str(x).strip() for x in (episode.get("participants", []) or []) if str(x).strip()]
|
||||
keywords = [str(x).strip() for x in (episode.get("keywords", []) or []) if str(x).strip()]
|
||||
if not participants:
|
||||
participants = self._collect_participants(ordered_hashes, limit=16)
|
||||
if not keywords:
|
||||
keywords = self._derive_keywords(sub_paragraphs, limit=12)
|
||||
|
||||
title = str(episode.get("title", "") or "").strip()[:120]
|
||||
summary = str(episode.get("summary", "") or "").strip()[:2000]
|
||||
if not title or not summary:
|
||||
continue
|
||||
|
||||
seed = json.dumps(
|
||||
{
|
||||
"source": source,
|
||||
"hashes": ordered_hashes,
|
||||
"version": segmentation_version,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
episode_id = compute_hash(seed)
|
||||
|
||||
payload = {
|
||||
"episode_id": episode_id,
|
||||
"source": source or None,
|
||||
"title": title,
|
||||
"summary": summary,
|
||||
"event_time_start": episode.get("event_time_start", event_start),
|
||||
"event_time_end": episode.get("event_time_end", event_end),
|
||||
"time_granularity": episode.get("time_granularity", granularity),
|
||||
"time_confidence": self._clamp_score(
|
||||
episode.get("time_confidence"),
|
||||
default=time_conf_default,
|
||||
),
|
||||
"participants": participants[:16],
|
||||
"keywords": keywords[:20],
|
||||
"evidence_ids": ordered_hashes,
|
||||
"paragraph_count": len(ordered_hashes),
|
||||
"llm_confidence": self._clamp_score(
|
||||
episode.get("llm_confidence"),
|
||||
default=0.0 if fallback_used else 0.6,
|
||||
),
|
||||
"segmentation_model": (
|
||||
str(episode.get("segmentation_model", "") or "").strip()
|
||||
or ("fallback_rule" if fallback_used else segmentation_model)
|
||||
),
|
||||
"segmentation_version": (
|
||||
str(episode.get("segmentation_version", "") or "").strip()
|
||||
or segmentation_version
|
||||
),
|
||||
}
|
||||
stored_payloads.append(payload)
|
||||
|
||||
return {
|
||||
"payloads": stored_payloads,
|
||||
"done_hashes": group_hashes,
|
||||
"episode_count": len(stored_payloads),
|
||||
"fallback_count": 1 if fallback_used else 0,
|
||||
}
|
||||
|
||||
async def process_group(self, group: Dict[str, Any]) -> Dict[str, Any]:
|
||||
result = await self._build_episode_payloads_for_group(group)
|
||||
stored_count = 0
|
||||
for payload in result.get("payloads") or []:
|
||||
stored = self.metadata_store.upsert_episode(payload)
|
||||
final_id = str(stored.get("episode_id") or payload.get("episode_id") or "")
|
||||
if final_id:
|
||||
self.metadata_store.bind_episode_paragraphs(
|
||||
final_id,
|
||||
list(payload.get("evidence_ids") or []),
|
||||
)
|
||||
stored_count += 1
|
||||
|
||||
result["episode_count"] = stored_count
|
||||
return {
|
||||
"done_hashes": list(result.get("done_hashes") or []),
|
||||
"episode_count": stored_count,
|
||||
"fallback_count": int(result.get("fallback_count") or 0),
|
||||
}
|
||||
|
||||
async def process_pending_rows(self, pending_rows: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
loaded, missing_hashes = self.load_pending_paragraphs(pending_rows)
|
||||
groups = self.group_paragraphs(loaded)
|
||||
|
||||
done_hashes: List[str] = list(missing_hashes)
|
||||
failed_hashes: Dict[str, str] = {}
|
||||
episode_count = 0
|
||||
fallback_count = 0
|
||||
|
||||
for group in groups:
|
||||
group_hashes = [str(p.get("hash", "") or "").strip() for p in (group.get("paragraphs") or [])]
|
||||
try:
|
||||
result = await self.process_group(group)
|
||||
done_hashes.extend(result.get("done_hashes") or [])
|
||||
episode_count += int(result.get("episode_count") or 0)
|
||||
fallback_count += int(result.get("fallback_count") or 0)
|
||||
except Exception as e:
|
||||
err = str(e)[:500]
|
||||
for h in group_hashes:
|
||||
if h:
|
||||
failed_hashes[h] = err
|
||||
|
||||
dedup_done = list(dict.fromkeys([h for h in done_hashes if h]))
|
||||
return {
|
||||
"done_hashes": dedup_done,
|
||||
"failed_hashes": failed_hashes,
|
||||
"episode_count": episode_count,
|
||||
"fallback_count": fallback_count,
|
||||
"missing_count": len(missing_hashes),
|
||||
"group_count": len(groups),
|
||||
}
|
||||
|
||||
async def rebuild_source(self, source: str) -> Dict[str, Any]:
|
||||
token = str(source or "").strip()
|
||||
if not token:
|
||||
return {
|
||||
"source": "",
|
||||
"episode_count": 0,
|
||||
"fallback_count": 0,
|
||||
"group_count": 0,
|
||||
"paragraph_count": 0,
|
||||
}
|
||||
|
||||
paragraphs = self.metadata_store.get_live_paragraphs_by_source(token)
|
||||
if not paragraphs:
|
||||
replace_result = self.metadata_store.replace_episodes_for_source(token, [])
|
||||
return {
|
||||
"source": token,
|
||||
"episode_count": int(replace_result.get("episode_count") or 0),
|
||||
"fallback_count": 0,
|
||||
"group_count": 0,
|
||||
"paragraph_count": 0,
|
||||
}
|
||||
|
||||
groups = self.group_paragraphs(paragraphs)
|
||||
payloads: List[Dict[str, Any]] = []
|
||||
fallback_count = 0
|
||||
|
||||
for group in groups:
|
||||
result = await self._build_episode_payloads_for_group(group)
|
||||
payloads.extend(list(result.get("payloads") or []))
|
||||
fallback_count += int(result.get("fallback_count") or 0)
|
||||
|
||||
replace_result = self.metadata_store.replace_episodes_for_source(token, payloads)
|
||||
return {
|
||||
"source": token,
|
||||
"episode_count": int(replace_result.get("episode_count") or 0),
|
||||
"fallback_count": fallback_count,
|
||||
"group_count": len(groups),
|
||||
"paragraph_count": len(paragraphs),
|
||||
}
|
||||
129
src/A_memorix/core/utils/hash.py
Normal file
129
src/A_memorix/core/utils/hash.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""
|
||||
哈希工具模块
|
||||
|
||||
提供文本哈希计算功能，用于唯一标识和去重。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from typing import Union
|
||||
|
||||
|
||||
def compute_hash(text: str, hash_type: str = "sha256") -> str:
|
||||
"""
|
||||
计算文本的哈希值
|
||||
|
||||
Args:
|
||||
text: 输入文本
|
||||
hash_type: 哈希算法类型（sha256, md5等）
|
||||
|
||||
Returns:
|
||||
哈希值字符串
|
||||
"""
|
||||
if hash_type == "sha256":
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
elif hash_type == "md5":
|
||||
return hashlib.md5(text.encode("utf-8")).hexdigest()
|
||||
else:
|
||||
raise ValueError(f"不支持的哈希算法: {hash_type}")
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""
|
||||
规范化文本用于哈希计算
|
||||
|
||||
执行以下操作：
|
||||
- 去除首尾空白
|
||||
- 统一换行符为\\n
|
||||
- 压缩多个连续空格
|
||||
- 去除不可见字符（保留换行和制表符）
|
||||
|
||||
Args:
|
||||
text: 输入文本
|
||||
|
||||
Returns:
|
||||
规范化后的文本
|
||||
"""
|
||||
# 去除首尾空白
|
||||
text = text.strip()
|
||||
|
||||
# 统一换行符
|
||||
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
|
||||
# 压缩多个连续空格为一个（但保留换行和制表符）
|
||||
text = re.sub(r"[^\S\n]+", " ", text)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def compute_paragraph_hash(paragraph: str) -> str:
|
||||
"""
|
||||
计算段落的哈希值
|
||||
|
||||
Args:
|
||||
paragraph: 段落文本
|
||||
|
||||
Returns:
|
||||
段落哈希值（用于paragraph-前缀）
|
||||
"""
|
||||
normalized = normalize_text(paragraph)
|
||||
return compute_hash(normalized)
|
||||
|
||||
|
||||
def compute_entity_hash(entity: str) -> str:
|
||||
"""
|
||||
计算实体的哈希值
|
||||
|
||||
Args:
|
||||
entity: 实体名称
|
||||
|
||||
Returns:
|
||||
实体哈希值（用于entity-前缀）
|
||||
"""
|
||||
normalized = entity.strip().lower()
|
||||
return compute_hash(normalized)
|
||||
|
||||
|
||||
def compute_relation_hash(relation: tuple) -> str:
|
||||
"""
|
||||
计算关系的哈希值
|
||||
|
||||
Args:
|
||||
relation: 关系元组 (subject, predicate, object)
|
||||
|
||||
Returns:
|
||||
关系哈希值（用于relation-前缀）
|
||||
"""
|
||||
# 将关系元组转为字符串
|
||||
relation_str = str(tuple(relation))
|
||||
return compute_hash(relation_str)
|
||||
|
||||
|
||||
def format_hash_key(hash_type: str, hash_value: str) -> str:
|
||||
"""
|
||||
格式化哈希键
|
||||
|
||||
Args:
|
||||
hash_type: 类型前缀（paragraph, entity, relation）
|
||||
hash_value: 哈希值
|
||||
|
||||
Returns:
|
||||
格式化的键（如 paragraph-abc123...）
|
||||
"""
|
||||
return f"{hash_type}-{hash_value}"
|
||||
|
||||
|
||||
def parse_hash_key(key: str) -> tuple[str, str]:
|
||||
"""
|
||||
解析哈希键
|
||||
|
||||
Args:
|
||||
key: 格式化的键（如 paragraph-abc123...）
|
||||
|
||||
Returns:
|
||||
(类型, 哈希值) 元组
|
||||
"""
|
||||
parts = key.split("-", 1)
|
||||
if len(parts) != 2:
|
||||
raise ValueError(f"无效的哈希键格式: {key}")
|
||||
return parts[0], parts[1]
|
||||
110
src/A_memorix/core/utils/import_payloads.py
Normal file
110
src/A_memorix/core/utils/import_payloads.py
Normal file
@@ -0,0 +1,110 @@
|
||||
"""Shared import payload normalization helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..storage import KnowledgeType, resolve_stored_knowledge_type
|
||||
from .time_parser import normalize_time_meta
|
||||
|
||||
|
||||
def _normalize_entities(raw_entities: Any) -> List[str]:
|
||||
if not isinstance(raw_entities, list):
|
||||
return []
|
||||
out: List[str] = []
|
||||
seen = set()
|
||||
for item in raw_entities:
|
||||
name = str(item or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
key = name.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_relations(raw_relations: Any) -> List[Dict[str, str]]:
|
||||
if not isinstance(raw_relations, list):
|
||||
return []
|
||||
out: List[Dict[str, str]] = []
|
||||
for item in raw_relations:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
subject = str(item.get("subject", "")).strip()
|
||||
predicate = str(item.get("predicate", "")).strip()
|
||||
obj = str(item.get("object", "")).strip()
|
||||
if not (subject and predicate and obj):
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"subject": subject,
|
||||
"predicate": predicate,
|
||||
"object": obj,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def normalize_paragraph_import_item(
|
||||
item: Any,
|
||||
*,
|
||||
default_source: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""Normalize one paragraph import item from text/json payloads."""
|
||||
|
||||
if isinstance(item, str):
|
||||
content = str(item)
|
||||
knowledge_type = resolve_stored_knowledge_type(None, content=content)
|
||||
return {
|
||||
"content": content,
|
||||
"knowledge_type": knowledge_type.value,
|
||||
"source": str(default_source or "").strip(),
|
||||
"time_meta": None,
|
||||
"entities": [],
|
||||
"relations": [],
|
||||
}
|
||||
|
||||
if not isinstance(item, dict) or "content" not in item:
|
||||
raise ValueError("段落项必须为字符串或包含 content 的对象")
|
||||
|
||||
content = str(item.get("content", "") or "")
|
||||
if not content.strip():
|
||||
raise ValueError("段落 content 不能为空")
|
||||
|
||||
raw_time_meta = {
|
||||
"event_time": item.get("event_time"),
|
||||
"event_time_start": item.get("event_time_start"),
|
||||
"event_time_end": item.get("event_time_end"),
|
||||
"time_range": item.get("time_range"),
|
||||
"time_granularity": item.get("time_granularity"),
|
||||
"time_confidence": item.get("time_confidence"),
|
||||
}
|
||||
time_meta_field = item.get("time_meta")
|
||||
if isinstance(time_meta_field, dict):
|
||||
raw_time_meta.update(time_meta_field)
|
||||
|
||||
knowledge_type_raw = item.get("knowledge_type")
|
||||
if knowledge_type_raw is None:
|
||||
knowledge_type_raw = item.get("type")
|
||||
knowledge_type = resolve_stored_knowledge_type(knowledge_type_raw, content=content)
|
||||
source = str(item.get("source") or default_source or "").strip()
|
||||
if not source:
|
||||
source = str(default_source or "").strip()
|
||||
|
||||
normalized_time_meta = normalize_time_meta(raw_time_meta)
|
||||
return {
|
||||
"content": content,
|
||||
"knowledge_type": knowledge_type.value,
|
||||
"source": source,
|
||||
"time_meta": normalized_time_meta if normalized_time_meta else None,
|
||||
"entities": _normalize_entities(item.get("entities")),
|
||||
"relations": _normalize_relations(item.get("relations")),
|
||||
}
|
||||
|
||||
|
||||
def normalize_summary_knowledge_type(value: Any) -> KnowledgeType:
|
||||
"""Normalize config-driven summary knowledge type."""
|
||||
|
||||
return resolve_stored_knowledge_type(value, content="")
|
||||
84
src/A_memorix/core/utils/io.py
Normal file
84
src/A_memorix/core/utils/io.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
IO Utilities
|
||||
|
||||
提供原子文件写入等IO辅助功能。
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import contextlib
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
@contextlib.contextmanager
|
||||
def atomic_write(file_path: Union[str, Path], mode: str = "w", encoding: str = None, **kwargs):
|
||||
"""
|
||||
原子文件写入上下文管理器
|
||||
|
||||
原理：
|
||||
1. 写入 .tmp 临时文件
|
||||
2. 写入成功后，使用 os.replace 原子替换目标文件
|
||||
3. 如果失败，自动删除临时文件
|
||||
|
||||
Args:
|
||||
file_path: 目标文件路径
|
||||
mode: 打开模式 ('w', 'wb' 等)
|
||||
encoding: 编码
|
||||
**kwargs: 传给 open() 的其他参数
|
||||
"""
|
||||
path = Path(file_path)
|
||||
# 确保父目录存在
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 临时文件路径
|
||||
tmp_path = path.with_suffix(path.suffix + ".tmp")
|
||||
|
||||
try:
|
||||
with open(tmp_path, mode, encoding=encoding, **kwargs) as f:
|
||||
yield f
|
||||
|
||||
# 确保写入磁盘
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
|
||||
# 原子替换 (Windows下可能需要先删除目标文件，但 os.replace 在 Py3.3+ 尽可能原子)
|
||||
# 注意: Windows 上如果有其他进程占用文件，os.replace 可能会失败
|
||||
os.replace(tmp_path, path)
|
||||
|
||||
except Exception as e:
|
||||
# 清理临时文件
|
||||
if tmp_path.exists():
|
||||
try:
|
||||
os.remove(tmp_path)
|
||||
except:
|
||||
pass
|
||||
raise e
|
||||
|
||||
@contextlib.contextmanager
|
||||
def atomic_save_path(file_path: Union[str, Path]):
|
||||
"""
|
||||
提供临时路径用于原子保存 (针对只接受路径的API，如Faiss)
|
||||
|
||||
Args:
|
||||
file_path: 最终目标路径
|
||||
|
||||
Yields:
|
||||
tmp_path: 临时文件路径 (str)
|
||||
"""
|
||||
path = Path(file_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = path.with_suffix(path.suffix + ".tmp")
|
||||
|
||||
try:
|
||||
yield str(tmp_path)
|
||||
|
||||
if Path(tmp_path).exists():
|
||||
os.replace(tmp_path, path)
|
||||
|
||||
except Exception as e:
|
||||
if Path(tmp_path).exists():
|
||||
try:
|
||||
os.remove(tmp_path)
|
||||
except:
|
||||
pass
|
||||
raise e
|
||||
89
src/A_memorix/core/utils/matcher.py
Normal file
89
src/A_memorix/core/utils/matcher.py
Normal file
@@ -0,0 +1,89 @@
|
||||
"""
|
||||
高效文本匹配工具模块
|
||||
|
||||
实现 Aho-Corasick 算法用于多模式匹配。
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Tuple, Set, Any
|
||||
from collections import deque
|
||||
|
||||
|
||||
class AhoCorasick:
|
||||
"""
|
||||
Aho-Corasick 自动机实现高效多模式匹配
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# next_states[state][char] = next_state
|
||||
self.next_states: List[Dict[str, int]] = [{}]
|
||||
# fail[state] = fail_state
|
||||
self.fail: List[int] = [0]
|
||||
# output[state] = set of patterns ending at this state
|
||||
self.output: List[Set[str]] = [set()]
|
||||
self.patterns: Set[str] = set()
|
||||
|
||||
def add_pattern(self, pattern: str):
|
||||
"""添加模式"""
|
||||
if not pattern:
|
||||
return
|
||||
self.patterns.add(pattern)
|
||||
state = 0
|
||||
for char in pattern:
|
||||
if char not in self.next_states[state]:
|
||||
new_state = len(self.next_states)
|
||||
self.next_states[state][char] = new_state
|
||||
self.next_states.append({})
|
||||
self.fail.append(0)
|
||||
self.output.append(set())
|
||||
state = self.next_states[state][char]
|
||||
self.output[state].add(pattern)
|
||||
|
||||
def build(self):
|
||||
"""构建失败指针"""
|
||||
queue = deque()
|
||||
# 处理第一层
|
||||
for char, state in self.next_states[0].items():
|
||||
queue.append(state)
|
||||
self.fail[state] = 0
|
||||
|
||||
while queue:
|
||||
r = queue.popleft()
|
||||
for char, s in self.next_states[r].items():
|
||||
queue.append(s)
|
||||
# 找到失败路径
|
||||
state = self.fail[r]
|
||||
while char not in self.next_states[state] and state != 0:
|
||||
state = self.fail[state]
|
||||
self.fail[s] = self.next_states[state].get(char, 0)
|
||||
# 合并输出
|
||||
self.output[s].update(self.output[self.fail[s]])
|
||||
|
||||
def search(self, text: str) -> List[Tuple[int, str]]:
|
||||
"""
|
||||
在文本中搜索所有模式
|
||||
|
||||
Returns:
|
||||
[(结束索引, 匹配到的模式), ...]
|
||||
"""
|
||||
state = 0
|
||||
results = []
|
||||
for i, char in enumerate(text):
|
||||
while char not in self.next_states[state] and state != 0:
|
||||
state = self.fail[state]
|
||||
state = self.next_states[state].get(char, 0)
|
||||
for pattern in self.output[state]:
|
||||
results.append((i, pattern))
|
||||
return results
|
||||
|
||||
def find_all(self, text: str) -> Dict[str, int]:
|
||||
"""
|
||||
查找并统计所有模式出现次数
|
||||
|
||||
Returns:
|
||||
{模式: 出现次数}
|
||||
"""
|
||||
results = self.search(text)
|
||||
stats = {}
|
||||
for _, pattern in results:
|
||||
stats[pattern] = stats.get(pattern, 0) + 1
|
||||
return stats
|
||||
189
src/A_memorix/core/utils/monitor.py
Normal file
189
src/A_memorix/core/utils/monitor.py
Normal file
@@ -0,0 +1,189 @@
|
||||
"""
|
||||
内存监控模块
|
||||
|
||||
提供内存使用监控和预警功能。
|
||||
"""
|
||||
|
||||
import gc
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable, Optional
|
||||
|
||||
try:
|
||||
import psutil
|
||||
HAS_PSUTIL = True
|
||||
except ImportError:
|
||||
HAS_PSUTIL = False
|
||||
|
||||
from src.common.logger import get_logger
|
||||
|
||||
logger = get_logger("A_Memorix.MemoryMonitor")
|
||||
|
||||
|
||||
class MemoryMonitor:
|
||||
"""
|
||||
内存监控器
|
||||
|
||||
功能：
|
||||
- 实时监控内存使用
|
||||
- 超过阈值时触发警告
|
||||
- 支持自动垃圾回收
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_memory_mb: int,
|
||||
warning_threshold: float = 0.9,
|
||||
check_interval: float = 10.0,
|
||||
enable_auto_gc: bool = True,
|
||||
):
|
||||
"""
|
||||
初始化内存监控器
|
||||
|
||||
Args:
|
||||
max_memory_mb: 最大内存限制（MB）
|
||||
warning_threshold: 警告阈值（0-1之间，默认0.9表示90%）
|
||||
check_interval: 检查间隔（秒）
|
||||
enable_auto_gc: 是否启用自动垃圾回收
|
||||
"""
|
||||
self.max_memory_mb = max_memory_mb
|
||||
self.warning_threshold = warning_threshold
|
||||
self.check_interval = check_interval
|
||||
self.enable_auto_gc = enable_auto_gc
|
||||
|
||||
self._running = False
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._callbacks: list[Callable[[float, float], None]] = []
|
||||
|
||||
def start(self):
|
||||
"""启动监控"""
|
||||
if self._running:
|
||||
logger.warning("内存监控已在运行")
|
||||
return
|
||||
|
||||
if not HAS_PSUTIL:
|
||||
logger.warning("psutil 未安装，内存监控功能不可用")
|
||||
return
|
||||
|
||||
self._running = True
|
||||
self._thread = threading.Thread(target=self._monitor_loop, daemon=True)
|
||||
self._thread.start()
|
||||
logger.info(f"内存监控已启动 (限制: {self.max_memory_mb}MB)")
|
||||
|
||||
def stop(self):
|
||||
"""停止监控"""
|
||||
if not self._running:
|
||||
return
|
||||
|
||||
self._running = False
|
||||
if self._thread:
|
||||
self._thread.join(timeout=5.0)
|
||||
logger.info("内存监控已停止")
|
||||
|
||||
def register_callback(self, callback: Callable[[float, float], None]):
|
||||
"""
|
||||
注册内存超限回调函数
|
||||
|
||||
Args:
|
||||
callback: 回调函数，接收 (当前使用MB, 限制MB) 参数
|
||||
"""
|
||||
self._callbacks.append(callback)
|
||||
|
||||
def get_current_memory_mb(self) -> float:
|
||||
"""
|
||||
获取当前进程内存使用量（MB）
|
||||
|
||||
Returns:
|
||||
内存使用量（MB）
|
||||
"""
|
||||
if not HAS_PSUTIL:
|
||||
# 降级方案：使用内置函数
|
||||
import sys
|
||||
return sys.getsizeof(gc.get_objects()) / 1024 / 1024
|
||||
|
||||
process = psutil.Process()
|
||||
return process.memory_info().rss / 1024 / 1024
|
||||
|
||||
def get_memory_usage_ratio(self) -> float:
|
||||
"""
|
||||
获取内存使用率
|
||||
|
||||
Returns:
|
||||
使用率（0-1之间）
|
||||
"""
|
||||
current = self.get_current_memory_mb()
|
||||
return current / self.max_memory_mb if self.max_memory_mb > 0 else 0
|
||||
|
||||
def _monitor_loop(self):
|
||||
"""监控循环"""
|
||||
while self._running:
|
||||
try:
|
||||
current_mb = self.get_current_memory_mb()
|
||||
ratio = current_mb / self.max_memory_mb if self.max_memory_mb > 0 else 0
|
||||
|
||||
# 检查是否超过阈值
|
||||
if ratio >= self.warning_threshold:
|
||||
logger.warning(
|
||||
f"内存使用率过高: {current_mb:.1f}MB / {self.max_memory_mb}MB "
|
||||
f"({ratio*100:.1f}%)"
|
||||
)
|
||||
|
||||
# 触发回调
|
||||
for callback in self._callbacks:
|
||||
try:
|
||||
callback(current_mb, self.max_memory_mb)
|
||||
except Exception as e:
|
||||
logger.error(f"内存回调执行失败: {e}")
|
||||
|
||||
# 自动垃圾回收
|
||||
if self.enable_auto_gc:
|
||||
before = self.get_current_memory_mb()
|
||||
gc.collect()
|
||||
after = self.get_current_memory_mb()
|
||||
freed = before - after
|
||||
if freed > 1: # 释放超过1MB才记录
|
||||
logger.info(f"垃圾回收释放: {freed:.1f}MB")
|
||||
|
||||
# 定期垃圾回收（即使未超限）
|
||||
elif self.enable_auto_gc and int(time.time()) % 60 == 0:
|
||||
gc.collect()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"内存监控出错: {e}")
|
||||
|
||||
# 等待下次检查
|
||||
time.sleep(self.check_interval)
|
||||
|
||||
def __enter__(self):
|
||||
"""上下文管理器入口"""
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""上下文管理器出口"""
|
||||
self.stop()
|
||||
|
||||
|
||||
def get_memory_info() -> dict:
|
||||
"""
|
||||
获取系统内存信息
|
||||
|
||||
Returns:
|
||||
内存信息字典
|
||||
"""
|
||||
if not HAS_PSUTIL:
|
||||
return {"error": "psutil 未安装"}
|
||||
|
||||
try:
|
||||
mem = psutil.virtual_memory()
|
||||
process = psutil.Process()
|
||||
|
||||
return {
|
||||
"system_total_gb": mem.total / 1024 / 1024 / 1024,
|
||||
"system_available_gb": mem.available / 1024 / 1024 / 1024,
|
||||
"system_usage_percent": mem.percent,
|
||||
"process_mb": process.memory_info().rss / 1024 / 1024,
|
||||
"process_percent": (process.memory_info().rss / mem.total) * 100,
|
||||
}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
165
src/A_memorix/core/utils/path_fallback_service.py
Normal file
165
src/A_memorix/core/utils/path_fallback_service.py
Normal file
@@ -0,0 +1,165 @@
|
||||
"""Shared path-fallback helpers for search post-processing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
from ..retrieval.dual_path import RetrievalResult
|
||||
|
||||
|
||||
def extract_entities(query: str, graph_store: Any) -> List[str]:
|
||||
"""Extract up to two graph nodes from a query using n-gram matching."""
|
||||
if not graph_store:
|
||||
return []
|
||||
|
||||
text = str(query or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# Keep the heuristic aligned with previous legacy behavior.
|
||||
tokens = (
|
||||
text.replace("?", " ")
|
||||
.replace("!", " ")
|
||||
.replace(".", " ")
|
||||
.split()
|
||||
)
|
||||
if not tokens:
|
||||
return []
|
||||
|
||||
found_entities = set()
|
||||
skip_indices = set()
|
||||
max_n = min(4, len(tokens))
|
||||
|
||||
for size in range(max_n, 0, -1):
|
||||
for i in range(len(tokens) - size + 1):
|
||||
if any(idx in skip_indices for idx in range(i, i + size)):
|
||||
continue
|
||||
span = " ".join(tokens[i : i + size])
|
||||
matched_node = graph_store.find_node(span, ignore_case=True)
|
||||
if not matched_node:
|
||||
continue
|
||||
found_entities.add(matched_node)
|
||||
for idx in range(i, i + size):
|
||||
skip_indices.add(idx)
|
||||
|
||||
return list(found_entities)
|
||||
|
||||
|
||||
def find_paths_between_entities(
|
||||
start_node: str,
|
||||
end_node: str,
|
||||
graph_store: Any,
|
||||
metadata_store: Any,
|
||||
*,
|
||||
max_depth: int = 3,
|
||||
max_paths: int = 5,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Find and enrich indirect paths between two nodes."""
|
||||
if not graph_store or not metadata_store:
|
||||
return []
|
||||
|
||||
try:
|
||||
paths = graph_store.find_paths(
|
||||
start_node,
|
||||
end_node,
|
||||
max_depth=max_depth,
|
||||
max_paths=max_paths,
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
if not paths:
|
||||
return []
|
||||
|
||||
edge_cache: Dict[Tuple[str, str], Tuple[str, str]] = {}
|
||||
formatted_paths: List[Dict[str, Any]] = []
|
||||
|
||||
for path_nodes in paths:
|
||||
if not isinstance(path_nodes, Sequence) or len(path_nodes) < 2:
|
||||
continue
|
||||
|
||||
path_desc: List[str] = []
|
||||
for i in range(len(path_nodes) - 1):
|
||||
u = str(path_nodes[i])
|
||||
v = str(path_nodes[i + 1])
|
||||
|
||||
cache_key = tuple(sorted((u, v)))
|
||||
if cache_key in edge_cache:
|
||||
pred, direction = edge_cache[cache_key]
|
||||
else:
|
||||
pred = "related"
|
||||
direction = "->"
|
||||
rels = metadata_store.get_relations(subject=u, object=v)
|
||||
if not rels:
|
||||
rels = metadata_store.get_relations(subject=v, object=u)
|
||||
direction = "<-"
|
||||
if rels:
|
||||
best_rel = max(rels, key=lambda x: x.get("confidence", 1.0))
|
||||
pred = str(best_rel.get("predicate", "related") or "related")
|
||||
edge_cache[cache_key] = (pred, direction)
|
||||
|
||||
step_str = f"-[{pred}]->" if direction == "->" else f"<-[{pred}]-"
|
||||
path_desc.append(step_str)
|
||||
|
||||
full_path_str = str(path_nodes[0])
|
||||
for i, step in enumerate(path_desc):
|
||||
full_path_str += f" {step} {path_nodes[i + 1]}"
|
||||
|
||||
formatted_paths.append(
|
||||
{
|
||||
"nodes": list(path_nodes),
|
||||
"description": full_path_str,
|
||||
}
|
||||
)
|
||||
|
||||
return formatted_paths
|
||||
|
||||
|
||||
def find_paths_from_query(
|
||||
query: str,
|
||||
graph_store: Any,
|
||||
metadata_store: Any,
|
||||
*,
|
||||
max_depth: int = 3,
|
||||
max_paths: int = 5,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Extract entities from query and resolve indirect paths."""
|
||||
entities = extract_entities(query, graph_store)
|
||||
if len(entities) != 2:
|
||||
return []
|
||||
return find_paths_between_entities(
|
||||
entities[0],
|
||||
entities[1],
|
||||
graph_store,
|
||||
metadata_store,
|
||||
max_depth=max_depth,
|
||||
max_paths=max_paths,
|
||||
)
|
||||
|
||||
|
||||
def to_retrieval_results(paths: Sequence[Dict[str, Any]]) -> List[RetrievalResult]:
|
||||
"""Convert path results into retrieval results for the unified pipeline."""
|
||||
converted: List[RetrievalResult] = []
|
||||
for item in paths:
|
||||
description = str(item.get("description", "")).strip()
|
||||
if not description:
|
||||
continue
|
||||
hash_seed = description.encode("utf-8")
|
||||
path_hash = f"path_{hashlib.sha1(hash_seed).hexdigest()}"
|
||||
converted.append(
|
||||
RetrievalResult(
|
||||
hash_value=path_hash,
|
||||
content=f"[Indirect Relation] {description}",
|
||||
score=0.95,
|
||||
result_type="relation",
|
||||
source="graph_path",
|
||||
metadata={
|
||||
"source": "graph_path",
|
||||
"is_indirect": True,
|
||||
"nodes": list(item.get("nodes", [])),
|
||||
},
|
||||
)
|
||||
)
|
||||
return converted
|
||||
|
||||
599
src/A_memorix/core/utils/person_profile_service.py
Normal file
599
src/A_memorix/core/utils/person_profile_service.py
Normal file
@@ -0,0 +1,599 @@
|
||||
"""
|
||||
人物画像服务
|
||||
|
||||
主链路：
|
||||
person_id -> 用户名/别名 -> 图谱关系 + 向量证据 -> 证据总结画像 -> 快照版本化存储
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlmodel import select
|
||||
|
||||
from src.common.logger import get_logger
|
||||
from src.common.database.database import get_db_session
|
||||
from src.common.database.database_model import PersonInfo
|
||||
|
||||
from ..embedding import EmbeddingAPIAdapter
|
||||
from ..retrieval import (
|
||||
DualPathRetriever,
|
||||
RetrievalStrategy,
|
||||
DualPathRetrieverConfig,
|
||||
SparseBM25Config,
|
||||
FusionConfig,
|
||||
GraphRelationRecallConfig,
|
||||
)
|
||||
from ..storage import MetadataStore, GraphStore, VectorStore
|
||||
|
||||
logger = get_logger("A_Memorix.PersonProfileService")
|
||||
|
||||
|
||||
class PersonProfileService:
|
||||
"""人物画像聚合/刷新服务。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata_store: MetadataStore,
|
||||
graph_store: Optional[GraphStore] = None,
|
||||
vector_store: Optional[VectorStore] = None,
|
||||
embedding_manager: Optional[EmbeddingAPIAdapter] = None,
|
||||
sparse_index: Any = None,
|
||||
plugin_config: Optional[dict] = None,
|
||||
retriever: Optional[DualPathRetriever] = None,
|
||||
):
|
||||
self.metadata_store = metadata_store
|
||||
self.graph_store = graph_store
|
||||
self.vector_store = vector_store
|
||||
self.embedding_manager = embedding_manager
|
||||
self.sparse_index = sparse_index
|
||||
self.plugin_config = plugin_config or {}
|
||||
self.retriever = retriever or self._build_retriever()
|
||||
|
||||
def _cfg(self, key: str, default: Any = None) -> Any:
|
||||
"""读取嵌套配置。"""
|
||||
if not isinstance(self.plugin_config, dict):
|
||||
return default
|
||||
current: Any = self.plugin_config
|
||||
for part in key.split("."):
|
||||
if isinstance(current, dict) and part in current:
|
||||
current = current[part]
|
||||
else:
|
||||
return default
|
||||
return current
|
||||
|
||||
def _build_retriever(self) -> Optional[DualPathRetriever]:
|
||||
"""按需构建检索器（无依赖时返回 None）。"""
|
||||
if not all(
|
||||
[
|
||||
self.vector_store is not None,
|
||||
self.graph_store is not None,
|
||||
self.metadata_store is not None,
|
||||
self.embedding_manager is not None,
|
||||
]
|
||||
):
|
||||
return None
|
||||
try:
|
||||
sparse_cfg_raw = self._cfg("retrieval.sparse", {}) or {}
|
||||
fusion_cfg_raw = self._cfg("retrieval.fusion", {}) or {}
|
||||
graph_recall_cfg_raw = self._cfg("retrieval.search.graph_recall", {}) or {}
|
||||
if not isinstance(sparse_cfg_raw, dict):
|
||||
sparse_cfg_raw = {}
|
||||
if not isinstance(fusion_cfg_raw, dict):
|
||||
fusion_cfg_raw = {}
|
||||
if not isinstance(graph_recall_cfg_raw, dict):
|
||||
graph_recall_cfg_raw = {}
|
||||
|
||||
sparse_cfg = SparseBM25Config(**sparse_cfg_raw)
|
||||
fusion_cfg = FusionConfig(**fusion_cfg_raw)
|
||||
graph_recall_cfg = GraphRelationRecallConfig(**graph_recall_cfg_raw)
|
||||
config = DualPathRetrieverConfig(
|
||||
top_k_paragraphs=int(self._cfg("retrieval.top_k_paragraphs", 20)),
|
||||
top_k_relations=int(self._cfg("retrieval.top_k_relations", 10)),
|
||||
top_k_final=int(self._cfg("retrieval.top_k_final", 10)),
|
||||
alpha=float(self._cfg("retrieval.alpha", 0.5)),
|
||||
enable_ppr=bool(self._cfg("retrieval.enable_ppr", True)),
|
||||
ppr_alpha=float(self._cfg("retrieval.ppr_alpha", 0.85)),
|
||||
ppr_concurrency_limit=int(self._cfg("retrieval.ppr_concurrency_limit", 4)),
|
||||
enable_parallel=bool(self._cfg("retrieval.enable_parallel", True)),
|
||||
retrieval_strategy=RetrievalStrategy.DUAL_PATH,
|
||||
debug=bool(self._cfg("advanced.debug", False)),
|
||||
sparse=sparse_cfg,
|
||||
fusion=fusion_cfg,
|
||||
graph_recall=graph_recall_cfg,
|
||||
)
|
||||
return DualPathRetriever(
|
||||
vector_store=self.vector_store,
|
||||
graph_store=self.graph_store,
|
||||
metadata_store=self.metadata_store,
|
||||
embedding_manager=self.embedding_manager,
|
||||
sparse_index=self.sparse_index,
|
||||
config=config,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"初始化人物画像检索器失败，将只使用关系证据: {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def resolve_person_id(identifier: str) -> str:
|
||||
"""按 person_id 或姓名/别名解析 person_id。"""
|
||||
if not identifier:
|
||||
return ""
|
||||
key = str(identifier).strip()
|
||||
if not key:
|
||||
return ""
|
||||
|
||||
try:
|
||||
with get_db_session(auto_commit=False) as session:
|
||||
record = session.exec(
|
||||
select(PersonInfo.person_id).where(PersonInfo.person_id == key).limit(1)
|
||||
).first()
|
||||
if record:
|
||||
return str(record)
|
||||
|
||||
record = session.exec(
|
||||
select(PersonInfo.person_id)
|
||||
.where(
|
||||
or_(
|
||||
PersonInfo.person_name == key,
|
||||
PersonInfo.user_nickname == key,
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
).first()
|
||||
if record:
|
||||
return str(record)
|
||||
|
||||
record = session.exec(
|
||||
select(PersonInfo.person_id)
|
||||
.where(PersonInfo.group_cardname.contains(key))
|
||||
.limit(1)
|
||||
).first()
|
||||
if record:
|
||||
return str(record)
|
||||
except Exception as e:
|
||||
logger.warning(f"按别名解析 person_id 失败: identifier={key}, err={e}")
|
||||
|
||||
if len(key) == 32 and all(ch in "0123456789abcdefABCDEF" for ch in key):
|
||||
return key.lower()
|
||||
|
||||
return ""
|
||||
|
||||
def _parse_group_nicks(self, raw_value: Any) -> List[str]:
|
||||
if not raw_value:
|
||||
return []
|
||||
if isinstance(raw_value, list):
|
||||
items = raw_value
|
||||
else:
|
||||
try:
|
||||
items = json.loads(raw_value)
|
||||
except Exception:
|
||||
return []
|
||||
names: List[str] = []
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
value = str(item.get("group_cardname") or item.get("group_nick_name") or "").strip()
|
||||
if value:
|
||||
names.append(value)
|
||||
elif isinstance(item, str):
|
||||
value = item.strip()
|
||||
if value:
|
||||
names.append(value)
|
||||
return names
|
||||
|
||||
def _parse_memory_traits(self, raw_value: Any) -> List[str]:
|
||||
if not raw_value:
|
||||
return []
|
||||
try:
|
||||
values = json.loads(raw_value) if isinstance(raw_value, str) else raw_value
|
||||
except Exception:
|
||||
return []
|
||||
if not isinstance(values, list):
|
||||
return []
|
||||
traits: List[str] = []
|
||||
for item in values:
|
||||
text = str(item).strip()
|
||||
if not text:
|
||||
continue
|
||||
if ":" in text:
|
||||
parts = text.split(":")
|
||||
if len(parts) >= 3:
|
||||
content = ":".join(parts[1:-1]).strip()
|
||||
if content:
|
||||
traits.append(content)
|
||||
continue
|
||||
traits.append(text)
|
||||
return traits[:10]
|
||||
|
||||
def _recover_aliases_from_memory(self, person_id: str) -> Tuple[List[str], str]:
|
||||
"""当人物主档案缺失时，从已有记忆证据里回捞可用别名。"""
|
||||
if not person_id:
|
||||
return [], ""
|
||||
|
||||
aliases: List[str] = []
|
||||
primary_name = ""
|
||||
seen = set()
|
||||
|
||||
try:
|
||||
paragraphs = self.metadata_store.get_paragraphs_by_entity(person_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"从记忆证据回捞人物别名失败: person_id={person_id}, err={e}")
|
||||
return [], ""
|
||||
|
||||
for paragraph in paragraphs[:20]:
|
||||
paragraph_hash = str(paragraph.get("hash", "") or "").strip()
|
||||
if not paragraph_hash:
|
||||
continue
|
||||
try:
|
||||
paragraph_entities = self.metadata_store.get_paragraph_entities(paragraph_hash)
|
||||
except Exception:
|
||||
paragraph_entities = []
|
||||
for entity in paragraph_entities:
|
||||
name = str(entity.get("name", "") or "").strip()
|
||||
if not name or name == person_id:
|
||||
continue
|
||||
key = name.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
aliases.append(name)
|
||||
if not primary_name:
|
||||
primary_name = name
|
||||
return aliases, primary_name
|
||||
|
||||
def get_person_aliases(self, person_id: str) -> Tuple[List[str], str, List[str]]:
|
||||
"""获取人物别名集合、主展示名、记忆特征。"""
|
||||
aliases: List[str] = []
|
||||
primary_name = ""
|
||||
memory_traits: List[str] = []
|
||||
if not person_id:
|
||||
return aliases, primary_name, memory_traits
|
||||
recovered_aliases, recovered_primary_name = self._recover_aliases_from_memory(person_id)
|
||||
try:
|
||||
with get_db_session(auto_commit=False) as session:
|
||||
record = session.exec(
|
||||
select(PersonInfo).where(PersonInfo.person_id == person_id).limit(1)
|
||||
).first()
|
||||
if not record:
|
||||
return recovered_aliases, recovered_primary_name or person_id, memory_traits
|
||||
person_name = str(getattr(record, "person_name", "") or "").strip()
|
||||
nickname = str(getattr(record, "user_nickname", "") or "").strip()
|
||||
group_nicks = self._parse_group_nicks(getattr(record, "group_cardname", None))
|
||||
memory_traits = self._parse_memory_traits(getattr(record, "memory_points", None))
|
||||
|
||||
primary_name = (
|
||||
person_name
|
||||
or nickname
|
||||
or recovered_primary_name
|
||||
or str(getattr(record, "user_id", "") or "").strip()
|
||||
or person_id
|
||||
)
|
||||
|
||||
candidates = [person_name, nickname] + group_nicks + recovered_aliases
|
||||
seen = set()
|
||||
for item in candidates:
|
||||
norm = str(item or "").strip()
|
||||
if not norm or norm in seen:
|
||||
continue
|
||||
seen.add(norm)
|
||||
aliases.append(norm)
|
||||
except Exception as e:
|
||||
logger.warning(f"解析人物别名失败: person_id={person_id}, err={e}")
|
||||
return aliases, primary_name, memory_traits
|
||||
|
||||
def _collect_relation_evidence(self, aliases: List[str], limit: int = 30) -> List[Dict[str, Any]]:
|
||||
relation_by_hash: Dict[str, Dict[str, Any]] = {}
|
||||
for alias in aliases:
|
||||
for rel in self.metadata_store.get_relations(subject=alias):
|
||||
h = str(rel.get("hash", ""))
|
||||
if h:
|
||||
relation_by_hash[h] = rel
|
||||
for rel in self.metadata_store.get_relations(object=alias):
|
||||
h = str(rel.get("hash", ""))
|
||||
if h:
|
||||
relation_by_hash[h] = rel
|
||||
|
||||
relations = list(relation_by_hash.values())
|
||||
relations.sort(key=lambda item: float(item.get("confidence", 0.0)), reverse=True)
|
||||
relations = relations[: max(1, int(limit))]
|
||||
|
||||
edges: List[Dict[str, Any]] = []
|
||||
for rel in relations:
|
||||
edges.append(
|
||||
{
|
||||
"hash": str(rel.get("hash", "")),
|
||||
"subject": str(rel.get("subject", "")),
|
||||
"predicate": str(rel.get("predicate", "")),
|
||||
"object": str(rel.get("object", "")),
|
||||
"confidence": float(rel.get("confidence", 1.0) or 1.0),
|
||||
}
|
||||
)
|
||||
return edges
|
||||
|
||||
def _collect_person_fact_evidence(self, person_id: str, limit: int = 4) -> List[Dict[str, Any]]:
|
||||
token = str(person_id or "").strip()
|
||||
if not token:
|
||||
return []
|
||||
|
||||
source = f"person_fact:{token}"
|
||||
paragraphs = [
|
||||
row
|
||||
for row in self.metadata_store.get_paragraphs_by_source(source)
|
||||
if not bool(row.get("is_deleted", 0))
|
||||
]
|
||||
paragraphs.sort(
|
||||
key=lambda item: float(item.get("updated_at") or item.get("created_at") or 0.0),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
evidence: List[Dict[str, Any]] = []
|
||||
for row in paragraphs[: max(1, int(limit))]:
|
||||
paragraph_hash = str(row.get("hash", "") or "")
|
||||
content = str(row.get("content", "") or "").strip()
|
||||
if not paragraph_hash or not content:
|
||||
continue
|
||||
evidence.append(
|
||||
{
|
||||
"hash": paragraph_hash,
|
||||
"type": "paragraph",
|
||||
"score": 1.1,
|
||||
"content": content[:220],
|
||||
"metadata": {},
|
||||
}
|
||||
)
|
||||
return evidence
|
||||
|
||||
async def _collect_vector_evidence(
|
||||
self,
|
||||
aliases: List[str],
|
||||
top_k: int = 12,
|
||||
person_id: str = "",
|
||||
) -> List[Dict[str, Any]]:
|
||||
alias_queries = [a for a in aliases if a]
|
||||
if not alias_queries and not person_id:
|
||||
return []
|
||||
|
||||
if self.retriever is None:
|
||||
# 回退：无检索器时只做简单内容匹配
|
||||
fallback: List[Dict[str, Any]] = []
|
||||
seen_hash = set()
|
||||
for alias in alias_queries:
|
||||
for para in self.metadata_store.search_paragraphs_by_content(alias)[: max(2, top_k // 2)]:
|
||||
h = str(para.get("hash", ""))
|
||||
if not h or h in seen_hash:
|
||||
continue
|
||||
seen_hash.add(h)
|
||||
fallback.append(
|
||||
{
|
||||
"hash": h,
|
||||
"type": "paragraph",
|
||||
"score": 0.0,
|
||||
"content": str(para.get("content", ""))[:180],
|
||||
"metadata": {},
|
||||
}
|
||||
)
|
||||
return fallback[:top_k]
|
||||
|
||||
per_alias_top_k = max(2, int(top_k / max(1, len(alias_queries))))
|
||||
seen_hash = set()
|
||||
evidence: List[Dict[str, Any]] = []
|
||||
for item in self._collect_person_fact_evidence(person_id, limit=max(2, min(4, top_k))):
|
||||
h = str(item.get("hash", "") or "")
|
||||
if not h or h in seen_hash:
|
||||
continue
|
||||
seen_hash.add(h)
|
||||
evidence.append(item)
|
||||
|
||||
for alias in alias_queries:
|
||||
try:
|
||||
results = await self.retriever.retrieve(alias, top_k=per_alias_top_k)
|
||||
except Exception as e:
|
||||
logger.warning(f"向量证据召回失败: alias={alias}, err={e}")
|
||||
continue
|
||||
for item in results:
|
||||
h = str(getattr(item, "hash_value", "") or "")
|
||||
if not h or h in seen_hash:
|
||||
continue
|
||||
seen_hash.add(h)
|
||||
evidence.append(
|
||||
{
|
||||
"hash": h,
|
||||
"type": str(getattr(item, "result_type", "")),
|
||||
"score": float(getattr(item, "score", 0.0) or 0.0),
|
||||
"content": str(getattr(item, "content", "") or "")[:220],
|
||||
"metadata": dict(getattr(item, "metadata", {}) or {}),
|
||||
}
|
||||
)
|
||||
evidence.sort(key=lambda x: x.get("score", 0.0), reverse=True)
|
||||
return evidence[:top_k]
|
||||
|
||||
def _build_profile_text(
|
||||
self,
|
||||
person_id: str,
|
||||
primary_name: str,
|
||||
aliases: List[str],
|
||||
relation_edges: List[Dict[str, Any]],
|
||||
vector_evidence: List[Dict[str, Any]],
|
||||
memory_traits: List[str],
|
||||
) -> str:
|
||||
"""基于证据构建画像文本（供 LLM 上下文注入）。"""
|
||||
lines: List[str] = []
|
||||
lines.append(f"人物ID: {person_id}")
|
||||
if primary_name:
|
||||
lines.append(f"主称呼: {primary_name}")
|
||||
if aliases:
|
||||
lines.append(f"别名: {', '.join(aliases[:8])}")
|
||||
if memory_traits:
|
||||
lines.append(f"记忆特征: {'; '.join(memory_traits[:6])}")
|
||||
|
||||
if relation_edges:
|
||||
lines.append("关系证据:")
|
||||
for rel in relation_edges[:6]:
|
||||
s = rel.get("subject", "")
|
||||
p = rel.get("predicate", "")
|
||||
o = rel.get("object", "")
|
||||
conf = float(rel.get("confidence", 0.0))
|
||||
lines.append(f"- {s} {p} {o} (conf={conf:.2f})")
|
||||
|
||||
if vector_evidence:
|
||||
lines.append("向量证据摘要:")
|
||||
for item in vector_evidence[:4]:
|
||||
content = str(item.get("content", "")).strip()
|
||||
if content:
|
||||
lines.append(f"- {content}")
|
||||
|
||||
if len(lines) <= 2:
|
||||
lines.append("暂无足够证据形成稳定画像。")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@staticmethod
|
||||
def _is_snapshot_stale(snapshot: Optional[Dict[str, Any]], ttl_seconds: float) -> bool:
|
||||
if not snapshot:
|
||||
return True
|
||||
now = time.time()
|
||||
expires_at = snapshot.get("expires_at")
|
||||
if expires_at is not None:
|
||||
try:
|
||||
return now >= float(expires_at)
|
||||
except Exception:
|
||||
return True
|
||||
updated_at = float(snapshot.get("updated_at") or 0.0)
|
||||
return (now - updated_at) >= ttl_seconds
|
||||
|
||||
def _apply_manual_override(self, person_id: str, profile_payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""将手工覆盖并入画像结果（覆盖 profile_text，同时保留 auto_profile_text）。"""
|
||||
payload = dict(profile_payload or {})
|
||||
auto_text = str(payload.get("profile_text", "") or "")
|
||||
payload["auto_profile_text"] = auto_text
|
||||
payload["has_manual_override"] = False
|
||||
payload["manual_override_text"] = ""
|
||||
payload["override_updated_at"] = None
|
||||
payload["override_updated_by"] = ""
|
||||
payload["profile_source"] = "auto_snapshot"
|
||||
|
||||
if not person_id or self.metadata_store is None:
|
||||
return payload
|
||||
|
||||
try:
|
||||
override = self.metadata_store.get_person_profile_override(person_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"读取人物画像手工覆盖失败: person_id={person_id}, err={e}")
|
||||
return payload
|
||||
|
||||
if not override:
|
||||
return payload
|
||||
|
||||
manual_text = str(override.get("override_text", "") or "").strip()
|
||||
if not manual_text:
|
||||
return payload
|
||||
|
||||
payload["has_manual_override"] = True
|
||||
payload["manual_override_text"] = manual_text
|
||||
payload["override_updated_at"] = override.get("updated_at")
|
||||
payload["override_updated_by"] = str(override.get("updated_by", "") or "")
|
||||
payload["profile_text"] = manual_text
|
||||
payload["profile_source"] = "manual_override"
|
||||
return payload
|
||||
|
||||
async def query_person_profile(
|
||||
self,
|
||||
person_id: str = "",
|
||||
person_keyword: str = "",
|
||||
top_k: int = 12,
|
||||
ttl_seconds: float = 6 * 3600,
|
||||
force_refresh: bool = False,
|
||||
source_note: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""查询或刷新人物画像。"""
|
||||
pid = str(person_id or "").strip()
|
||||
if not pid and person_keyword:
|
||||
pid = self.resolve_person_id(person_keyword)
|
||||
|
||||
if not pid:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "person_id 无效，且未能通过别名解析",
|
||||
}
|
||||
|
||||
latest = self.metadata_store.get_latest_person_profile_snapshot(pid)
|
||||
if not force_refresh and not self._is_snapshot_stale(latest, ttl_seconds):
|
||||
aliases, primary_name, _ = self.get_person_aliases(pid)
|
||||
payload = {
|
||||
"success": True,
|
||||
"person_id": pid,
|
||||
"person_name": primary_name,
|
||||
"from_cache": True,
|
||||
**(latest or {}),
|
||||
}
|
||||
if aliases and not payload.get("aliases"):
|
||||
payload["aliases"] = aliases
|
||||
return {
|
||||
**self._apply_manual_override(pid, payload),
|
||||
}
|
||||
|
||||
aliases, primary_name, memory_traits = self.get_person_aliases(pid)
|
||||
if not aliases and person_keyword:
|
||||
aliases = [person_keyword.strip()]
|
||||
primary_name = person_keyword.strip()
|
||||
relation_edges = self._collect_relation_evidence(aliases, limit=max(10, top_k * 2))
|
||||
vector_evidence = await self._collect_vector_evidence(aliases, top_k=max(4, top_k), person_id=pid)
|
||||
|
||||
evidence_ids = [
|
||||
str(item.get("hash", ""))
|
||||
for item in (relation_edges + vector_evidence)
|
||||
if str(item.get("hash", "")).strip()
|
||||
]
|
||||
dedup_ids: List[str] = []
|
||||
seen = set()
|
||||
for item in evidence_ids:
|
||||
if item in seen:
|
||||
continue
|
||||
seen.add(item)
|
||||
dedup_ids.append(item)
|
||||
|
||||
profile_text = self._build_profile_text(
|
||||
person_id=pid,
|
||||
primary_name=primary_name,
|
||||
aliases=aliases,
|
||||
relation_edges=relation_edges,
|
||||
vector_evidence=vector_evidence,
|
||||
memory_traits=memory_traits,
|
||||
)
|
||||
|
||||
expires_at = time.time() + float(ttl_seconds) if ttl_seconds > 0 else None
|
||||
snapshot = self.metadata_store.upsert_person_profile_snapshot(
|
||||
person_id=pid,
|
||||
profile_text=profile_text,
|
||||
aliases=aliases,
|
||||
relation_edges=relation_edges,
|
||||
vector_evidence=vector_evidence,
|
||||
evidence_ids=dedup_ids,
|
||||
expires_at=expires_at,
|
||||
source_note=source_note,
|
||||
)
|
||||
payload = {
|
||||
"success": True,
|
||||
"person_id": pid,
|
||||
"person_name": primary_name,
|
||||
"from_cache": False,
|
||||
**snapshot,
|
||||
}
|
||||
return {
|
||||
**self._apply_manual_override(pid, payload),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def format_persona_profile_block(profile: Dict[str, Any]) -> str:
|
||||
"""格式化给 replyer 的注入块。"""
|
||||
if not profile or not profile.get("success"):
|
||||
return ""
|
||||
text = str(profile.get("profile_text", "") or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
return (
|
||||
"【人物画像-内部参考】\n"
|
||||
f"{text}\n"
|
||||
"仅供内部推理，不要向用户逐字复述。"
|
||||
)
|
||||
27
src/A_memorix/core/utils/plugin_id_policy.py
Normal file
27
src/A_memorix/core/utils/plugin_id_policy.py
Normal file
@@ -0,0 +1,27 @@
|
||||
"""Plugin ID matching policy for A_Memorix."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
class PluginIdPolicy:
|
||||
"""Centralized plugin id normalization/matching policy."""
|
||||
|
||||
CANONICAL_ID = "a_memorix"
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, plugin_id: Any) -> str:
|
||||
if not isinstance(plugin_id, str):
|
||||
return ""
|
||||
return plugin_id.strip().lower()
|
||||
|
||||
@classmethod
|
||||
def is_target_plugin_id(cls, plugin_id: Any) -> bool:
|
||||
normalized = cls.normalize(plugin_id)
|
||||
if not normalized:
|
||||
return False
|
||||
if normalized == cls.CANONICAL_ID:
|
||||
return True
|
||||
return normalized.split(".")[-1] == cls.CANONICAL_ID
|
||||
|
||||
344
src/A_memorix/core/utils/quantization.py
Normal file
344
src/A_memorix/core/utils/quantization.py
Normal file
@@ -0,0 +1,344 @@
|
||||
"""
|
||||
向量量化工具模块
|
||||
|
||||
提供向量量化与反量化功能，用于压缩存储空间。
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
from enum import Enum
|
||||
from typing import Tuple, Union
|
||||
|
||||
from src.common.logger import get_logger
|
||||
|
||||
logger = get_logger("A_Memorix.Quantization")
|
||||
|
||||
|
||||
class QuantizationType(Enum):
|
||||
"""量化类型枚举"""
|
||||
FLOAT32 = "float32" # 无量化
|
||||
INT8 = "int8" # 标量量化（8位整数）
|
||||
PQ = "pq" # 乘积量化（Product Quantization）
|
||||
|
||||
|
||||
def quantize_vector(
|
||||
vector: np.ndarray,
|
||||
quant_type: QuantizationType = QuantizationType.INT8,
|
||||
) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]:
|
||||
"""
|
||||
量化向量
|
||||
|
||||
Args:
|
||||
vector: 输入向量（float32）
|
||||
quant_type: 量化类型
|
||||
|
||||
Returns:
|
||||
量化后的向量：
|
||||
- INT8: int8向量
|
||||
- PQ: (编码向量, 聚类中心) 元组
|
||||
"""
|
||||
if quant_type == QuantizationType.FLOAT32:
|
||||
return vector.astype(np.float32)
|
||||
|
||||
elif quant_type == QuantizationType.INT8:
|
||||
return _scalar_quantize_int8(vector)
|
||||
|
||||
elif quant_type == QuantizationType.PQ:
|
||||
return _product_quantize(vector)
|
||||
|
||||
else:
|
||||
raise ValueError(f"不支持的量化类型: {quant_type}")
|
||||
|
||||
|
||||
def dequantize_vector(
|
||||
quantized_vector: Union[np.ndarray, Tuple[np.ndarray, np.ndarray]],
|
||||
quant_type: QuantizationType = QuantizationType.INT8,
|
||||
original_shape: Tuple[int, ...] = None,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
反量化向量
|
||||
|
||||
Args:
|
||||
quantized_vector: 量化后的向量
|
||||
quant_type: 量化类型
|
||||
original_shape: 原始向量形状（用于PQ）
|
||||
|
||||
Returns:
|
||||
反量化后的向量（float32）
|
||||
"""
|
||||
if quant_type == QuantizationType.FLOAT32:
|
||||
return quantized_vector.astype(np.float32)
|
||||
|
||||
elif quant_type == QuantizationType.INT8:
|
||||
return _scalar_dequantize_int8(quantized_vector)
|
||||
|
||||
elif quant_type == QuantizationType.PQ:
|
||||
if not isinstance(quantized_vector, tuple):
|
||||
raise ValueError("PQ反量化需要列表/元组格式: (codes, centroids)")
|
||||
return _product_dequantize(quantized_vector[0], quantized_vector[1])
|
||||
|
||||
else:
|
||||
raise ValueError(f"不支持的量化类型: {quant_type}")
|
||||
|
||||
|
||||
def _scalar_quantize_int8(vector: np.ndarray) -> np.ndarray:
|
||||
"""
|
||||
标量量化：float32 -> int8
|
||||
|
||||
将向量归一化到 [0, 255] 范围，然后映射到 int8
|
||||
|
||||
Args:
|
||||
vector: 输入向量
|
||||
|
||||
Returns:
|
||||
量化后的 int8 向量
|
||||
"""
|
||||
# 计算最小最大值
|
||||
min_val = np.min(vector)
|
||||
max_val = np.max(vector)
|
||||
|
||||
# 避免除零
|
||||
if max_val == min_val:
|
||||
return np.zeros_like(vector, dtype=np.int8)
|
||||
|
||||
# 归一化到 [0, 255]
|
||||
normalized = (vector - min_val) / (max_val - min_val) * 255
|
||||
|
||||
# 映射到 [-128, 127] 并转换为 int8
|
||||
# np.round might return float, minus 128 then cast
|
||||
quantized = np.round(normalized - 128.0).astype(np.int8)
|
||||
|
||||
# 存储归一化参数（用于反量化）
|
||||
# 在实际存储中，这些参数需要单独保存
|
||||
# 这里为了简单，我们使用一个全局字典来模拟
|
||||
if not hasattr(_scalar_quantize_int8, "_params"):
|
||||
_scalar_quantize_int8._params = {}
|
||||
|
||||
vector_id = id(vector)
|
||||
_scalar_quantize_int8._params[vector_id] = (min_val, max_val)
|
||||
|
||||
return quantized
|
||||
|
||||
|
||||
def _scalar_dequantize_int8(quantized: np.ndarray) -> np.ndarray:
|
||||
"""
|
||||
标量反量化：int8 -> float32
|
||||
|
||||
Args:
|
||||
quantized: 量化后的 int8 向量
|
||||
|
||||
Returns:
|
||||
反量化后的 float32 向量
|
||||
"""
|
||||
# 计算归一化参数（如果提供了）
|
||||
# 在实际应用中，min_val 和 max_val 应该被保存
|
||||
if not hasattr(_scalar_dequantize_int8, "_params"):
|
||||
# 默认假设范围是 [-1, 1]
|
||||
return (quantized.astype(np.float32) + 128.0) / 255.0 * 2.0 - 1.0
|
||||
|
||||
# 尝试查找参数 (这里只是演示逻辑，实际应从存储中读取)
|
||||
# return (quantized.astype(np.float32) + 128.0) / 255.0 * (max - min) + min
|
||||
return (quantized.astype(np.float32) + 128.0) / 255.0
|
||||
|
||||
|
||||
def quantize_matrix(
|
||||
matrix: np.ndarray,
|
||||
quant_type: QuantizationType = QuantizationType.INT8,
|
||||
) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]:
|
||||
"""
|
||||
量化矩阵（批量量化向量）
|
||||
|
||||
Args:
|
||||
matrix: 输入矩阵（N x D，每行是一个向量）
|
||||
quant_type: 量化类型
|
||||
|
||||
Returns:
|
||||
量化后的矩阵
|
||||
"""
|
||||
if quant_type == QuantizationType.FLOAT32:
|
||||
return matrix.astype(np.float32)
|
||||
|
||||
elif quant_type == QuantizationType.INT8:
|
||||
# 对整个矩阵进行全局归一化
|
||||
min_val = np.min(matrix)
|
||||
max_val = np.max(matrix)
|
||||
|
||||
if max_val == min_val:
|
||||
return np.zeros_like(matrix, dtype=np.int8)
|
||||
|
||||
# 归一化到 [0, 255]
|
||||
normalized = (matrix - min_val) / (max_val - min_val) * 255
|
||||
quantized = np.round(normalized).astype(np.int8)
|
||||
|
||||
return quantized
|
||||
|
||||
else:
|
||||
raise ValueError(f"不支持的量化类型: {quant_type}")
|
||||
|
||||
|
||||
def dequantize_matrix(
|
||||
quantized_matrix: np.ndarray,
|
||||
quant_type: QuantizationType = QuantizationType.INT8,
|
||||
min_val: float = None,
|
||||
max_val: float = None,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
反量化矩阵
|
||||
|
||||
Args:
|
||||
quantized_matrix: 量化后的矩阵
|
||||
quant_type: 量化类型
|
||||
min_val: 归一化最小值（int8反量化需要）
|
||||
max_val: 归一化最大值（int8反量化需要）
|
||||
|
||||
Returns:
|
||||
反量化后的矩阵
|
||||
"""
|
||||
if quant_type == QuantizationType.FLOAT32:
|
||||
return quantized_matrix.astype(np.float32)
|
||||
|
||||
elif quant_type == QuantizationType.INT8:
|
||||
# 使用提供的归一化参数反量化
|
||||
if min_val is None or max_val is None:
|
||||
# 默认假设范围是 [0, 255] -> [-1, 1]
|
||||
return quantized_matrix.astype(np.float32) / 127.0
|
||||
else:
|
||||
# 恢复到原始范围
|
||||
normalized = quantized_matrix.astype(np.float32) / 255.0
|
||||
return normalized * (max_val - min_val) + min_val
|
||||
|
||||
else:
|
||||
raise ValueError(f"不支持的量化类型: {quant_type}")
|
||||
|
||||
|
||||
def estimate_memory_reduction(
|
||||
num_vectors: int,
|
||||
dimension: int,
|
||||
from_type: QuantizationType,
|
||||
to_type: QuantizationType,
|
||||
) -> Tuple[float, float]:
|
||||
"""
|
||||
估算内存节省量
|
||||
|
||||
Args:
|
||||
num_vectors: 向量数量
|
||||
dimension: 向量维度
|
||||
from_type: 原始量化类型
|
||||
to_type: 目标量化类型
|
||||
|
||||
Returns:
|
||||
(原始大小MB, 量化后大小MB, 节省比例)
|
||||
"""
|
||||
# 计算每个向量占用的字节数
|
||||
bytes_per_element = {
|
||||
QuantizationType.FLOAT32: 4,
|
||||
QuantizationType.INT8: 1,
|
||||
QuantizationType.PQ: 0.25, # 假设压缩到1/4
|
||||
}
|
||||
|
||||
original_bytes = num_vectors * dimension * bytes_per_element[from_type]
|
||||
quantized_bytes = num_vectors * dimension * bytes_per_element[to_type]
|
||||
|
||||
original_mb = original_bytes / 1024 / 1024
|
||||
quantized_mb = quantized_bytes / 1024 / 1024
|
||||
reduction_ratio = (original_bytes - quantized_bytes) / original_bytes
|
||||
|
||||
return original_mb, quantized_mb, reduction_ratio
|
||||
|
||||
|
||||
def estimate_compression_stats(
|
||||
num_vectors: int,
|
||||
dimension: int,
|
||||
quant_type: QuantizationType,
|
||||
) -> dict:
|
||||
"""
|
||||
估算压缩统计信息
|
||||
|
||||
Args:
|
||||
num_vectors: 向量数量
|
||||
dimension: 向量维度
|
||||
quant_type: 量化类型
|
||||
|
||||
Returns:
|
||||
统计信息字典
|
||||
"""
|
||||
original_mb, quantized_mb, ratio = estimate_memory_reduction(
|
||||
num_vectors, dimension, QuantizationType.FLOAT32, quant_type
|
||||
)
|
||||
|
||||
return {
|
||||
"num_vectors": num_vectors,
|
||||
"dimension": dimension,
|
||||
"quantization_type": quant_type.value,
|
||||
"original_size_mb": round(original_mb, 2),
|
||||
"quantized_size_mb": round(quantized_mb, 2),
|
||||
"saved_mb": round(original_mb - quantized_mb, 2),
|
||||
"compression_ratio": round(ratio * 100, 2),
|
||||
}
|
||||
|
||||
|
||||
def _product_quantize(
|
||||
vector: np.ndarray, m: int = 8, k: int = 256
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""
|
||||
乘积量化 (PQ) 简化实现
|
||||
|
||||
Args:
|
||||
vector: 输入向量 (D,)
|
||||
m: 子空间数量
|
||||
k: 每个子空间的聚类中心数
|
||||
|
||||
Returns:
|
||||
(编码后的向量, 聚类中心)
|
||||
"""
|
||||
d = vector.shape[0]
|
||||
if d % m != 0:
|
||||
raise ValueError(f"维度 {d} 必须能被子空间数量 {m} 整除")
|
||||
|
||||
ds = d // m # 子空间维度
|
||||
codes = np.zeros(m, dtype=np.uint8)
|
||||
centroids = np.zeros((m, k, ds), dtype=np.float32)
|
||||
|
||||
# 这里采用一种简化的 PQ：不进行 K-means 训练，
|
||||
# 而是预定一些量化点或针对单向量的微型聚类（实际应用中应离线训练）
|
||||
# 为了演示，我们直接将子空间切分为 k 份进行量化
|
||||
for i in range(m):
|
||||
sub_vec = vector[i * ds : (i + 1) * ds]
|
||||
# 简化：假定每个子空间的取值范围并划分
|
||||
# 实际 PQ 应使用 k-means 产生的 centroids
|
||||
# 这里为演示创建一个随机 codebook 并找到最接近的核心
|
||||
sub_min, sub_max = np.min(sub_vec), np.max(sub_vec)
|
||||
if sub_max == sub_min:
|
||||
linspace = np.zeros(k)
|
||||
else:
|
||||
linspace = np.linspace(sub_min, sub_max, k)
|
||||
|
||||
for j in range(k):
|
||||
centroids[i, j, :] = linspace[j]
|
||||
|
||||
# 编码：这里简化为取子空间均值找最接近的 centroid
|
||||
sub_mean = np.mean(sub_vec)
|
||||
code = np.argmin(np.abs(linspace - sub_mean))
|
||||
codes[i] = code
|
||||
|
||||
return codes, centroids
|
||||
|
||||
|
||||
def _product_dequantize(codes: np.ndarray, centroids: np.ndarray) -> np.ndarray:
|
||||
"""
|
||||
PQ 反量化
|
||||
|
||||
Args:
|
||||
codes: 编码向量 (M,)
|
||||
centroids: 聚类中心 (M, K, DS)
|
||||
|
||||
Returns:
|
||||
恢复后的向量 (D,)
|
||||
"""
|
||||
m, k, ds = centroids.shape
|
||||
vector = np.zeros(m * ds, dtype=np.float32)
|
||||
|
||||
for i in range(m):
|
||||
code = codes[i]
|
||||
vector[i * ds : (i + 1) * ds] = centroids[i, code, :]
|
||||
|
||||
return vector
|
||||
121
src/A_memorix/core/utils/relation_query.py
Normal file
121
src/A_memorix/core/utils/relation_query.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""关系查询规格解析工具。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class RelationQuerySpec:
|
||||
raw: str
|
||||
is_structured: bool
|
||||
subject: Optional[str]
|
||||
predicate: Optional[str]
|
||||
object: Optional[str]
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
_NATURAL_LANGUAGE_PATTERN = re.compile(
|
||||
r"(^\s*(what|who|which|how|why|when|where)\b|"
|
||||
r"\?|？|"
|
||||
r"\b(relation|related|between)\b|"
|
||||
r"(什么关系|有哪些关系|之间|关联))",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_natural_language(raw: str) -> bool:
|
||||
text = str(raw or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
return _NATURAL_LANGUAGE_PATTERN.search(text) is not None
|
||||
|
||||
|
||||
def parse_relation_query_spec(relation_spec: str) -> RelationQuerySpec:
|
||||
raw = str(relation_spec or "").strip()
|
||||
if not raw:
|
||||
return RelationQuerySpec(
|
||||
raw=raw,
|
||||
is_structured=False,
|
||||
subject=None,
|
||||
predicate=None,
|
||||
object=None,
|
||||
error="empty",
|
||||
)
|
||||
|
||||
if "|" in raw:
|
||||
parts = [p.strip() for p in raw.split("|")]
|
||||
if len(parts) < 2:
|
||||
return RelationQuerySpec(
|
||||
raw=raw,
|
||||
is_structured=True,
|
||||
subject=None,
|
||||
predicate=None,
|
||||
object=None,
|
||||
error="invalid_pipe_format",
|
||||
)
|
||||
return RelationQuerySpec(
|
||||
raw=raw,
|
||||
is_structured=True,
|
||||
subject=parts[0] or None,
|
||||
predicate=parts[1] or None,
|
||||
object=parts[2] if len(parts) > 2 and parts[2] else None,
|
||||
)
|
||||
|
||||
if "->" in raw:
|
||||
parts = [p.strip() for p in raw.split("->") if p.strip()]
|
||||
if len(parts) >= 3:
|
||||
return RelationQuerySpec(
|
||||
raw=raw,
|
||||
is_structured=True,
|
||||
subject=parts[0],
|
||||
predicate=parts[1],
|
||||
object=parts[2],
|
||||
)
|
||||
if len(parts) == 2:
|
||||
return RelationQuerySpec(
|
||||
raw=raw,
|
||||
is_structured=True,
|
||||
subject=parts[0],
|
||||
predicate=None,
|
||||
object=parts[1],
|
||||
)
|
||||
return RelationQuerySpec(
|
||||
raw=raw,
|
||||
is_structured=True,
|
||||
subject=None,
|
||||
predicate=None,
|
||||
object=None,
|
||||
error="invalid_arrow_format",
|
||||
)
|
||||
|
||||
if _looks_like_natural_language(raw):
|
||||
return RelationQuerySpec(
|
||||
raw=raw,
|
||||
is_structured=False,
|
||||
subject=None,
|
||||
predicate=None,
|
||||
object=None,
|
||||
)
|
||||
|
||||
# 仅保留低歧义的紧凑三元组作为兼容语法，例如 "Alice likes Apple"。
|
||||
# 两词形式过于模糊，不再视为结构化关系查询。
|
||||
parts = raw.split()
|
||||
if len(parts) == 3:
|
||||
return RelationQuerySpec(
|
||||
raw=raw,
|
||||
is_structured=True,
|
||||
subject=parts[0],
|
||||
predicate=parts[1],
|
||||
object=parts[2],
|
||||
)
|
||||
|
||||
return RelationQuerySpec(
|
||||
raw=raw,
|
||||
is_structured=False,
|
||||
subject=None,
|
||||
predicate=None,
|
||||
object=None,
|
||||
)
|
||||
166
src/A_memorix/core/utils/relation_write_service.py
Normal file
166
src/A_memorix/core/utils/relation_write_service.py
Normal file
@@ -0,0 +1,166 @@
|
||||
"""
|
||||
统一关系写入与关系向量化服务。
|
||||
|
||||
规则：
|
||||
1. 元数据是主数据源，向量是从索引。
|
||||
2. 关系先写 metadata，再写向量。
|
||||
3. 向量失败不回滚 metadata，依赖状态机与回填任务修复。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from src.common.logger import get_logger
|
||||
|
||||
|
||||
logger = get_logger("A_Memorix.RelationWriteService")
|
||||
|
||||
|
||||
@dataclass
|
||||
class RelationWriteResult:
|
||||
hash_value: str
|
||||
vector_written: bool
|
||||
vector_already_exists: bool
|
||||
vector_state: str
|
||||
|
||||
|
||||
class RelationWriteService:
|
||||
"""关系写入收口服务。"""
|
||||
|
||||
ERROR_MAX_LEN = 500
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata_store: Any,
|
||||
graph_store: Any,
|
||||
vector_store: Any,
|
||||
embedding_manager: Any,
|
||||
):
|
||||
self.metadata_store = metadata_store
|
||||
self.graph_store = graph_store
|
||||
self.vector_store = vector_store
|
||||
self.embedding_manager = embedding_manager
|
||||
|
||||
@staticmethod
|
||||
def build_relation_vector_text(subject: str, predicate: str, obj: str) -> str:
|
||||
s = str(subject or "").strip()
|
||||
p = str(predicate or "").strip()
|
||||
o = str(obj or "").strip()
|
||||
# 双表达：兼容关键词检索与自然语言问句
|
||||
return f"{s} {p} {o}\n{s}和{o}的关系是{p}"
|
||||
|
||||
async def ensure_relation_vector(
|
||||
self,
|
||||
hash_value: str,
|
||||
subject: str,
|
||||
predicate: str,
|
||||
obj: str,
|
||||
*,
|
||||
max_error_len: int = ERROR_MAX_LEN,
|
||||
) -> RelationWriteResult:
|
||||
"""
|
||||
为已有关系确保向量存在并更新状态。
|
||||
"""
|
||||
if hash_value in self.vector_store:
|
||||
self.metadata_store.set_relation_vector_state(hash_value, "ready")
|
||||
return RelationWriteResult(
|
||||
hash_value=hash_value,
|
||||
vector_written=False,
|
||||
vector_already_exists=True,
|
||||
vector_state="ready",
|
||||
)
|
||||
|
||||
self.metadata_store.set_relation_vector_state(hash_value, "pending")
|
||||
try:
|
||||
vector_text = self.build_relation_vector_text(subject, predicate, obj)
|
||||
embedding = await self.embedding_manager.encode(vector_text)
|
||||
self.vector_store.add(
|
||||
vectors=embedding.reshape(1, -1),
|
||||
ids=[hash_value],
|
||||
)
|
||||
self.metadata_store.set_relation_vector_state(hash_value, "ready")
|
||||
logger.info(
|
||||
"metric.relation_vector_write_success=1 "
|
||||
"metric.relation_vector_write_success_count=1 "
|
||||
f"hash={hash_value[:16]}"
|
||||
)
|
||||
return RelationWriteResult(
|
||||
hash_value=hash_value,
|
||||
vector_written=True,
|
||||
vector_already_exists=False,
|
||||
vector_state="ready",
|
||||
)
|
||||
except ValueError:
|
||||
# 向量已存在冲突，按成功处理
|
||||
self.metadata_store.set_relation_vector_state(hash_value, "ready")
|
||||
return RelationWriteResult(
|
||||
hash_value=hash_value,
|
||||
vector_written=False,
|
||||
vector_already_exists=True,
|
||||
vector_state="ready",
|
||||
)
|
||||
except Exception as e:
|
||||
err = str(e)[:max_error_len]
|
||||
self.metadata_store.set_relation_vector_state(
|
||||
hash_value,
|
||||
"failed",
|
||||
error=err,
|
||||
bump_retry=True,
|
||||
)
|
||||
logger.warning(
|
||||
"metric.relation_vector_write_fail=1 "
|
||||
"metric.relation_vector_write_fail_count=1 "
|
||||
f"hash={hash_value[:16]} "
|
||||
f"err={err}"
|
||||
)
|
||||
return RelationWriteResult(
|
||||
hash_value=hash_value,
|
||||
vector_written=False,
|
||||
vector_already_exists=False,
|
||||
vector_state="failed",
|
||||
)
|
||||
|
||||
async def upsert_relation_with_vector(
|
||||
self,
|
||||
subject: str,
|
||||
predicate: str,
|
||||
obj: str,
|
||||
confidence: float = 1.0,
|
||||
source_paragraph: str = "",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
*,
|
||||
write_vector: bool = True,
|
||||
) -> RelationWriteResult:
|
||||
"""
|
||||
统一关系写入：
|
||||
1) 写 metadata relation
|
||||
2) 写 graph edge relation_hash
|
||||
3) 按需写 relation vector
|
||||
"""
|
||||
rel_hash = self.metadata_store.add_relation(
|
||||
subject=subject,
|
||||
predicate=predicate,
|
||||
obj=obj,
|
||||
confidence=confidence,
|
||||
source_paragraph=source_paragraph,
|
||||
metadata=metadata or {},
|
||||
)
|
||||
self.graph_store.add_edges([(subject, obj)], relation_hashes=[rel_hash])
|
||||
|
||||
if not write_vector:
|
||||
self.metadata_store.set_relation_vector_state(rel_hash, "none")
|
||||
return RelationWriteResult(
|
||||
hash_value=rel_hash,
|
||||
vector_written=False,
|
||||
vector_already_exists=False,
|
||||
vector_state="none",
|
||||
)
|
||||
|
||||
return await self.ensure_relation_vector(
|
||||
hash_value=rel_hash,
|
||||
subject=subject,
|
||||
predicate=predicate,
|
||||
obj=obj,
|
||||
)
|
||||
1858
src/A_memorix/core/utils/retrieval_tuning_manager.py
Normal file
1858
src/A_memorix/core/utils/retrieval_tuning_manager.py
Normal file
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user