Version: 0.9.76.dev.260505

后端:
1.阶段 6 agent / memory 服务化收口
- 新增 cmd/agent 独立进程入口,承载 agent zrpc server、agent outbox relay / consumer 和运行时依赖初始化
- 补齐 services/agent/rpc 的 Chat stream 与 conversation meta/list/timeline、schedule-preview、context-stats、schedule-state unary RPC
- 新增 gateway/client/agent 与 shared/contracts/agent,将 /api/v1/agent chat 和非 chat 门面切到 agent zrpc
- 收缩 gateway 本地 AgentService 装配,双 RPC 开关开启时不再初始化本地 agent 编排、LLM、RAG 和 memory reader fallback
- 将 backend/memory 物理迁入 services/memory,私有实现收入 internal,保留 module/model/observe 作为 memory 服务门面
- 调整 memory outbox、memory reader 和 agent 记忆渲染链路的 import 与服务边界,cmd/memory 独占 memory worker / consumer
- 关闭 gateway 侧 agent outbox worker 所有权,agent relay / consumer 由 cmd/agent 独占,gateway 仅保留 HTTP/SSE 门面与迁移期开关回退
- 更新阶段 6 文档,记录 agent / memory 当前切流点、smoke 结果,以及 backend/client 与 gateway/shared 的目录收口口径
This commit is contained in:
Losita
2026-05-05 19:31:39 +08:00
parent d7184b776b
commit 2a96f4c6f9
72 changed files with 2775 additions and 291 deletions

View File

@@ -0,0 +1,73 @@
package cleanup
import (
"sort"
"time"
"github.com/LoveLosita/smartflow/backend/model"
)
const dedupRecentTieWindow = 24 * time.Hour
// DedupDecision 描述单个重复组的治理结论。
type DedupDecision struct {
Keep model.MemoryItem
Archive []model.MemoryItem
}
// DecideDedupGroup 决定一组重复 active 记忆中“保留谁、归档谁”。
//
// 步骤化说明:
// 1. 先按“最近更新时间”判断谁更值得保留,符合治理计划里的“优先保留最近更新”;
// 2. 若更新时间非常接近,再比较 confidence/importance避免刚好相差几秒就误保留低质量版本
// 3. 最后用主键逆序兜底,保证同组治理结果稳定可复现。
func DecideDedupGroup(items []model.MemoryItem) DedupDecision {
if len(items) == 0 {
return DedupDecision{}
}
ordered := make([]model.MemoryItem, len(items))
copy(ordered, items)
sort.SliceStable(ordered, func(i, j int) bool {
return preferDedupKeep(ordered[i], ordered[j])
})
return DedupDecision{
Keep: ordered[0],
Archive: ordered[1:],
}
}
func preferDedupKeep(left model.MemoryItem, right model.MemoryItem) bool {
leftTime := dedupBaseTime(left)
rightTime := dedupBaseTime(right)
diff := leftTime.Sub(rightTime)
if diff < 0 {
diff = -diff
}
if diff > dedupRecentTieWindow {
return leftTime.After(rightTime)
}
if left.Confidence != right.Confidence {
return left.Confidence > right.Confidence
}
if left.Importance != right.Importance {
return left.Importance > right.Importance
}
if !leftTime.Equal(rightTime) {
return leftTime.After(rightTime)
}
return left.ID > right.ID
}
func dedupBaseTime(item model.MemoryItem) time.Time {
if item.UpdatedAt != nil {
return *item.UpdatedAt
}
if item.CreatedAt != nil {
return *item.CreatedAt
}
return time.Time{}
}

View File

@@ -0,0 +1,257 @@
package cleanup
import (
"context"
"errors"
"strconv"
"strings"
"time"
"github.com/LoveLosita/smartflow/backend/model"
memoryrepo "github.com/LoveLosita/smartflow/backend/services/memory/internal/repo"
memoryutils "github.com/LoveLosita/smartflow/backend/services/memory/internal/utils"
memoryvectorsync "github.com/LoveLosita/smartflow/backend/services/memory/internal/vectorsync"
memoryobserve "github.com/LoveLosita/smartflow/backend/services/memory/observe"
"gorm.io/gorm"
)
// DedupRunner 负责执行一次离线记忆去重治理。
//
// 职责边界:
// 1. 只处理“active + content_hash 非空”的重复组;
// 2. 只负责 archive + audit + 向量删除桥接,不负责自动定时调度;
// 3. 支持 dry-run便于上线初期先观察治理结果再正式落库。
type DedupRunner struct {
db *gorm.DB
itemRepo *memoryrepo.ItemRepo
auditRepo *memoryrepo.AuditRepo
vectorSyncer *memoryvectorsync.Syncer
observer memoryobserve.Observer
metrics memoryobserve.MetricsRecorder
}
func NewDedupRunner(
db *gorm.DB,
itemRepo *memoryrepo.ItemRepo,
auditRepo *memoryrepo.AuditRepo,
vectorSyncer *memoryvectorsync.Syncer,
observer memoryobserve.Observer,
metrics memoryobserve.MetricsRecorder,
) *DedupRunner {
if observer == nil {
observer = memoryobserve.NewNopObserver()
}
if metrics == nil {
metrics = memoryobserve.NewNopMetrics()
}
return &DedupRunner{
db: db,
itemRepo: itemRepo,
auditRepo: auditRepo,
vectorSyncer: vectorSyncer,
observer: observer,
metrics: metrics,
}
}
// Run 执行一次离线去重治理。
func (r *DedupRunner) Run(ctx context.Context, req model.MemoryDedupCleanupRequest) (model.MemoryDedupCleanupResult, error) {
result := model.MemoryDedupCleanupResult{
DryRun: req.DryRun,
}
if r == nil || r.db == nil || r.itemRepo == nil || r.auditRepo == nil {
return result, errors.New("memory dedup runner is not initialized")
}
items, err := r.itemRepo.ListActiveItemsForDedup(ctx, req.UserID, req.Limit)
if err != nil {
r.recordDedupObserve(ctx, req, result, false, err)
return result, err
}
groups := groupDuplicateItems(items)
result.ScannedGroupCount = len(groups)
if len(groups) == 0 {
r.recordDedupObserve(ctx, req, result, true, nil)
return result, nil
}
for _, group := range groups {
decision := DecideDedupGroup(group)
if decision.Keep.ID > 0 {
result.KeptCount++
}
if len(decision.Archive) == 0 {
continue
}
result.DedupedGroupCount++
archiveIDs := collectDedupIDs(decision.Archive)
result.ArchivedCount += len(archiveIDs)
result.ArchivedIDs = append(result.ArchivedIDs, archiveIDs...)
if req.DryRun {
continue
}
now := time.Now()
txErr := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
itemRepo := r.itemRepo.WithTx(tx)
auditRepo := r.auditRepo.WithTx(tx)
if archiveErr := itemRepo.ArchiveByIDsAt(ctx, archiveIDs, now); archiveErr != nil {
return archiveErr
}
for _, item := range decision.Archive {
after := item
after.Status = model.MemoryItemStatusArchived
after.UpdatedAt = &now
after.VectorStatus = "pending"
audit := memoryutils.BuildItemAuditLog(
item.ID,
item.UserID,
memoryutils.AuditOperationArchive,
normalizeCleanupOperator(req.OperatorType),
normalizeCleanupReason(req.Reason),
&item,
&after,
)
if createErr := auditRepo.Create(ctx, audit); createErr != nil {
return createErr
}
}
return nil
})
if txErr != nil {
r.recordDedupObserve(ctx, req, result, false, txErr)
return result, txErr
}
r.vectorSyncer.Delete(ctx, "", archiveIDs)
r.metrics.AddCounter(memoryobserve.MetricCleanupArchivedTotal, int64(len(archiveIDs)), map[string]string{
"dry_run": "false",
})
}
r.recordDedupObserve(ctx, req, result, true, nil)
return result, nil
}
func groupDuplicateItems(items []model.MemoryItem) [][]model.MemoryItem {
if len(items) == 0 {
return nil
}
result := make([][]model.MemoryItem, 0)
currentGroup := make([]model.MemoryItem, 0, 2)
currentKey := ""
for _, item := range items {
key := dedupGroupKey(item)
if key == "" {
continue
}
if currentKey == "" || currentKey != key {
if len(currentGroup) > 1 {
copied := make([]model.MemoryItem, len(currentGroup))
copy(copied, currentGroup)
result = append(result, copied)
}
currentKey = key
currentGroup = currentGroup[:0]
}
currentGroup = append(currentGroup, item)
}
if len(currentGroup) > 1 {
copied := make([]model.MemoryItem, len(currentGroup))
copy(copied, currentGroup)
result = append(result, copied)
}
return result
}
func dedupGroupKey(item model.MemoryItem) string {
contentHash := strings.TrimSpace(derefString(item.ContentHash))
if item.UserID <= 0 || strings.TrimSpace(item.MemoryType) == "" || contentHash == "" {
return ""
}
return strings.Join([]string{
strconv.Itoa(item.UserID),
item.MemoryType,
contentHash,
}, "::")
}
func collectDedupIDs(items []model.MemoryItem) []int64 {
ids := make([]int64, 0, len(items))
for _, item := range items {
if item.ID <= 0 {
continue
}
ids = append(ids, item.ID)
}
return ids
}
func normalizeCleanupOperator(operatorType string) string {
operatorType = strings.TrimSpace(operatorType)
if operatorType == "" {
return "system"
}
return memoryutils.NormalizeOperatorType(operatorType)
}
func normalizeCleanupReason(reason string) string {
reason = strings.TrimSpace(reason)
if reason == "" {
return "离线 dedup 治理归档重复记忆"
}
return reason
}
func derefString(value *string) string {
if value == nil {
return ""
}
return strings.TrimSpace(*value)
}
func (r *DedupRunner) recordDedupObserve(
ctx context.Context,
req model.MemoryDedupCleanupRequest,
result model.MemoryDedupCleanupResult,
success bool,
err error,
) {
if r == nil {
return
}
status := "success"
level := memoryobserve.LevelInfo
if !success || err != nil {
status = "error"
level = memoryobserve.LevelWarn
}
r.observer.Observe(ctx, memoryobserve.Event{
Level: level,
Component: memoryobserve.ComponentCleanup,
Operation: memoryobserve.OperationDedup,
Fields: map[string]any{
"user_id": req.UserID,
"limit": req.Limit,
"dry_run": req.DryRun,
"scanned_group_count": result.ScannedGroupCount,
"deduped_group_count": result.DedupedGroupCount,
"archived_count": result.ArchivedCount,
"success": success && err == nil,
"error": err,
"error_code": memoryobserve.ClassifyError(err),
},
})
r.metrics.AddCounter(memoryobserve.MetricCleanupRunTotal, 1, map[string]string{
"operation": "dedup",
"status": status,
})
}

View File

@@ -0,0 +1,135 @@
package orchestrator
import (
"context"
"fmt"
"log"
"strings"
llmservice "github.com/LoveLosita/smartflow/backend/services/llm"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
const defaultDecisionCompareMaxTokens = 600
// LLMDecisionOrchestrator 负责对"一条新 fact vs 一条旧记忆"做关系判断。
//
// 职责边界:
// 1. 每次只比较一对,是最小粒度的 LLM 调用;
// 2. LLM 只输出 relation关系类型不输出 action不输出 target ID
// 3. LLM 调用失败时返回 error由上层决定是否视为 unrelated。
type LLMDecisionOrchestrator struct {
client *llmservice.Client
cfg memorymodel.Config
logger *log.Logger
}
// NewLLMDecisionOrchestrator 构造决策比对编排器。
func NewLLMDecisionOrchestrator(client *llmservice.Client, cfg memorymodel.Config) *LLMDecisionOrchestrator {
return &LLMDecisionOrchestrator{
client: client,
cfg: cfg,
logger: log.Default(),
}
}
// Compare 对单条新 fact 与单条旧候选做关系判断。
//
// 返回语义:
// 1. 成功时返回比对结果relation 为四种合法值之一;
// 2. LLM 不可用或输出异常时返回 error上层应视为 unrelated
// 3. 不做最终决策,最终动作由确定性汇总逻辑产出。
func (o *LLMDecisionOrchestrator) Compare(
ctx context.Context,
fact memorymodel.NormalizedFact,
candidate memorymodel.CandidateSnapshot,
) (*memorymodel.ComparisonResult, error) {
if o == nil || o.client == nil {
return nil, fmt.Errorf("决策编排器未初始化")
}
// 1. 构建逐对比较 prompt极简二元判断LLM 只输出 relation。
systemPrompt := buildDecisionCompareSystemPrompt()
userPrompt := buildDecisionCompareUserPrompt(fact, candidate)
messages := llmservice.BuildSystemUserMessages(systemPrompt, nil, userPrompt)
// 2. 调用 LLM 做结构化输出,温度用低值保证判断稳定。
resp, _, err := llmservice.GenerateJSON[decisionCompareResponse](
ctx,
o.client,
messages,
llmservice.GenerateOptions{
Temperature: 0.1,
MaxTokens: defaultDecisionCompareMaxTokens,
Thinking: resolveMemoryThinkingMode(o.cfg.LLMThinking),
},
)
if err != nil {
if o.logger != nil {
o.logger.Printf("[WARN][去重] 决策比对 LLM 调用失败: memory_type=%s candidate_id=%d err=%v", fact.MemoryType, candidate.MemoryID, err)
}
return nil, err
}
// 3. 映射 LLM 输出到 ComparisonResultMemoryID 由代码填充而非 LLM。
result := &memorymodel.ComparisonResult{
MemoryID: candidate.MemoryID,
Relation: normalizeRelation(resp.Relation),
UpdatedContent: strings.TrimSpace(resp.UpdatedContent),
UpdatedTitle: strings.TrimSpace(resp.UpdatedTitle),
Reason: strings.TrimSpace(resp.Reason),
}
return result, nil
}
// decisionCompareResponse 是 LLM 逐对比较的 JSON 输出结构。
type decisionCompareResponse struct {
Relation string `json:"relation"`
UpdatedContent string `json:"updated_content"`
UpdatedTitle string `json:"updated_title"`
Reason string `json:"reason"`
}
// normalizeRelation 统一 relation 字段为小写标准形式。
func normalizeRelation(raw string) string {
return strings.ToLower(strings.TrimSpace(raw))
}
// buildDecisionCompareSystemPrompt 构建逐对比较的系统 prompt。
func buildDecisionCompareSystemPrompt() string {
return strings.TrimSpace(`你是一个记忆关系判断器。请判断"新事实"和"旧记忆"之间的关系。
关系类型:
- duplicate两者表达相同意思新事实没有新信息
- update新事实是对旧记忆的修正、补充或更精确表述
- conflict新事实与旧记忆在同一话题上存在矛盾如"喜欢X"变为"不喜欢X"、"去了A地"变为"实际去了B地"),旧记忆已过时
- unrelated两者说的是不同的事情或属于同一大类下的不同偏好如"喜欢唱歌"与"喜欢打球"是不同爱好,不矛盾)
输出 JSON
{"relation":"...","updated_content":"...","updated_title":"...","reason":"..."}
规则:
1. relation=update 时updated_content 必须写出合并后的完整内容(不是只写差异部分)
2. 其余 relation 类型updated_content 留空即可
3. reason 写简短判断依据
4. 只输出 JSON不要输出解释或 markdown
5. conflict 仅限同一话题内的矛盾信息;不同话题的偏好、不同领域的兴趣一律判 unrelated`)
}
// buildDecisionCompareUserPrompt 构建逐对比较的用户 prompt。
func buildDecisionCompareUserPrompt(fact memorymodel.NormalizedFact, candidate memorymodel.CandidateSnapshot) string {
return fmt.Sprintf("新事实:【%s】%s\n旧记忆【%s】%s",
fact.MemoryType, fact.Content,
candidate.MemoryType, candidate.Content,
)
}
// resolveMemoryThinkingMode 根据配置布尔值返回对应的 ThinkingMode。
func resolveMemoryThinkingMode(enabled bool) llmservice.ThinkingMode {
if enabled {
return llmservice.ThinkingModeEnabled
}
return llmservice.ThinkingModeDisabled
}

View File

@@ -0,0 +1,331 @@
package orchestrator
import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
llmservice "github.com/LoveLosita/smartflow/backend/services/llm"
memoryutils "github.com/LoveLosita/smartflow/backend/services/memory/internal/utils"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
const (
defaultMemoryExtractMaxTokens = 1200
defaultMemoryExtractMaxFacts = 5
)
// LLMWriteOrchestrator 负责把单条对话消息转成可入库的记忆候选。
//
// 职责边界:
// 1. 负责调用 LLM 做抽取、把输出标准化成 memory_facts
// 2. 不负责落库,不负责任务状态机推进;
// 3. 当 LLM 不可用或输出异常时,回退到保守的本地抽取,保证链路不完全断。
type LLMWriteOrchestrator struct {
client *llmservice.Client
cfg memorymodel.Config
logger *log.Logger
}
// NewLLMWriteOrchestrator 构造 LLM 版记忆写入编排器。
func NewLLMWriteOrchestrator(client *llmservice.Client, cfg memorymodel.Config) *LLMWriteOrchestrator {
return &LLMWriteOrchestrator{
client: client,
cfg: cfg,
logger: log.Default(),
}
}
// ExtractFacts 从单条消息中抽取可入库事实。
//
// 返回语义:
// 1. 成功时返回标准化后的候选事实;
// 2. 即使 LLM 失败,也尽量返回保守的 fallback 结果,避免 worker 空转报错;
// 3. 只有输入本身为空时才返回空结果。
func (o *LLMWriteOrchestrator) ExtractFacts(ctx context.Context, payload memorymodel.ExtractJobPayload) ([]memorymodel.NormalizedFact, error) {
sourceText := strings.TrimSpace(payload.SourceText)
if sourceText == "" {
return nil, nil
}
if o == nil || o.client == nil {
return fallbackNormalizedFacts(payload), nil
}
messages := llmservice.BuildSystemUserMessages(
buildMemoryExtractSystemPrompt(o.cfg.ExtractPrompt),
nil,
buildMemoryExtractUserPrompt(payload),
)
resp, rawResult, err := llmservice.GenerateJSON[memoryExtractResponse](
ctx,
o.client,
messages,
llmservice.GenerateOptions{
Temperature: clampTemperature(o.cfg.LLMTemperature),
MaxTokens: defaultMemoryExtractMaxTokens,
Thinking: resolveMemoryThinkingMode(o.cfg.LLMThinking),
Metadata: map[string]any{
"stage": "memory_extract",
"user_id": payload.UserID,
"conversation_id": payload.ConversationID,
},
},
)
if err != nil {
if o.logger != nil {
o.logger.Printf("[WARN] memory extract llm failed user_id=%d conversation_id=%s err=%v raw=%s",
payload.UserID, payload.ConversationID, err, truncateForLog(rawResult))
}
return fallbackNormalizedFacts(payload), nil
}
facts := convertExtractResponse(resp)
normalized := memoryutils.NormalizeFacts(facts)
if len(normalized) == 0 {
return fallbackNormalizedFacts(payload), nil
}
return normalized, nil
}
type memoryExtractResponse struct {
MessageIntent string `json:"message_intent"`
Facts []memoryExtractFact `json:"facts"`
}
type memoryExtractFact struct {
MemoryType string `json:"memory_type"`
Title string `json:"title"`
Content string `json:"content"`
Confidence float64 `json:"confidence"`
Importance float64 `json:"importance"`
SensitivityLevel int `json:"sensitivity_level"`
IsExplicit bool `json:"is_explicit"`
}
type memoryExtractPromptInput struct {
UserID int `json:"user_id"`
ConversationID string `json:"conversation_id"`
AssistantID string `json:"assistant_id,omitempty"`
RunID string `json:"run_id,omitempty"`
SourceMessageID int64 `json:"source_message_id,omitempty"`
SourceRole string `json:"source_role"`
SourceText string `json:"source_text"`
OccurredAt string `json:"occurred_at"`
TraceID string `json:"trace_id,omitempty"`
}
func buildMemoryExtractSystemPrompt(override string) string {
override = strings.TrimSpace(override)
if override != "" {
return override
}
return strings.TrimSpace(`你是一个”记忆守门员”。
你的任务是判断用户消息是否包含值得长期记住的信息,如有则提取。
请只输出 JSON 对象,不要输出解释、不要输出 markdown。
输出格式:
{
“message_intent”: “chitchat|task_request|knowledge_qa|preference|personal_fact|standing_instruction”,
“facts”: [
{
“memory_type”: “preference|constraint|fact”,
“title”: “短标题”,
“content”: “完整事实内容”,
“confidence”: 0.0,
“importance”: 0.0,
“sensitivity_level”: 0,
“is_explicit”: false
}
]
}
意图分类规则:
- chitchat闲聊、寒暄、情绪表达”你好””谢谢””我今天好累””嗯嗯”
- task_request一次性任务请求”帮我查天气””定个闹钟””帮我写个邮件”
- knowledge_qa知识问答、信息查询”什么是量子力学””北京明天多少度”
- preference用户偏好、习惯、口味”我喜欢吃辣””别用简称””我习惯用微信”
- personal_fact个人事实”我有两个孩子””我在上海工作””我老婆对花生过敏”
- standing_instruction持久指令”以后都用英文回复我””记住我的生日是3月5号”
规则:
1. 先判断 message_intent。chitchat / task_request / knowledge_qa 三类facts 输出空数组。
2. 只有 preference / personal_fact / standing_instruction 才提取 facts最多 3 条。
3. 一条消息可能同时包含任务和偏好(如”帮我查天气,记住我喜欢晴天”),此时 intent 取偏好类型facts 只保留偏好部分。
4. confidence 表示这条事实是否真的值得长期记,取 0 到 1。低于 0.5 的不要输出。
5. importance 表示对后续陪伴的价值,取 0 到 1。
6. sensitivity_level 取 0 到 2数字越大越敏感。
7. 用户明确说”记住”或”以后提醒我”时is_explicit 设为 true。
8. 宁可漏记也不要滥记。大多数消息不应该产生任何 facts。`)
}
func buildMemoryExtractUserPrompt(payload memorymodel.ExtractJobPayload) string {
request := memoryExtractPromptInput{
UserID: payload.UserID,
ConversationID: payload.ConversationID,
AssistantID: payload.AssistantID,
RunID: payload.RunID,
SourceMessageID: payload.SourceMessageID,
SourceRole: payload.SourceRole,
SourceText: payload.SourceText,
OccurredAt: payload.OccurredAt.Format("2006-01-02 15:04:05"),
TraceID: payload.TraceID,
}
raw, err := json.MarshalIndent(request, "", " ")
if err != nil {
return fmt.Sprintf("请分析这条用户消息,判断是否需要写入长期记忆:%s", payload.SourceText)
}
return fmt.Sprintf("请分析下面这条用户消息,判断 message_intent如包含值得长期记住的信息则提取 facts。\n输入\n%s",
string(raw))
}
func convertExtractResponse(resp *memoryExtractResponse) []memorymodel.FactCandidate {
if resp == nil {
return nil
}
// 意图过滤:跳过不需要记忆的消息类型。
// 兼容自定义 prompt不返回 message_intent 时跳过此检查,保持向后兼容)。
if intent := strings.TrimSpace(resp.MessageIntent); intent != "" {
if isSkipIntent(intent) {
return nil
}
}
if len(resp.Facts) == 0 {
return nil
}
result := make([]memorymodel.FactCandidate, 0, len(resp.Facts))
for _, fact := range resp.Facts {
memoryType := memorymodel.NormalizeMemoryType(fact.MemoryType)
if memoryType == "" {
continue
}
content := strings.TrimSpace(fact.Content)
if content == "" {
continue
}
confidence := clamp01(fact.Confidence)
if confidence == 0 {
confidence = 0.6
}
importance := clamp01(fact.Importance)
if importance == 0 {
importance = defaultImportanceByType(memoryType)
}
result = append(result, memorymodel.FactCandidate{
MemoryType: memoryType,
Title: strings.TrimSpace(fact.Title),
Content: content,
Confidence: confidence,
Importance: importance,
SensitivityLevel: clampInt(fact.SensitivityLevel, 0, 2),
IsExplicit: fact.IsExplicit,
})
}
return result
}
func fallbackNormalizedFacts(payload memorymodel.ExtractJobPayload) []memorymodel.NormalizedFact {
sourceText := strings.TrimSpace(payload.SourceText)
if sourceText == "" {
return nil
}
return memoryutils.NormalizeFacts([]memorymodel.FactCandidate{
{
MemoryType: memorymodel.MemoryTypeFact,
Title: buildFallbackTitle(sourceText),
Content: sourceText,
Confidence: 0.45,
Importance: defaultImportanceByType(memorymodel.MemoryTypeFact),
SensitivityLevel: 0,
IsExplicit: false,
},
})
}
func buildFallbackTitle(sourceText string) string {
runes := []rune(strings.TrimSpace(sourceText))
if len(runes) == 0 {
return "用户提到"
}
if len(runes) > 24 {
runes = runes[:24]
}
return "用户提到:" + string(runes)
}
func clampTemperature(v float64) float64 {
if v <= 0 {
return 0.1
}
if v > 1 {
return 1
}
return v
}
func clamp01(v float64) float64 {
if v < 0 {
return 0
}
if v > 1 {
return 1
}
return v
}
func clampInt(v, minValue, maxValue int) int {
if v < minValue {
return minValue
}
if v > maxValue {
return maxValue
}
return v
}
func defaultImportanceByType(memoryType string) float64 {
switch memoryType {
case memorymodel.MemoryTypePreference:
return 0.85
case memorymodel.MemoryTypeConstraint:
return 0.95
default:
return 0.6
}
}
// isSkipIntent 判断意图是否属于"不需要记忆"的类别。
// chitchat / task_request / knowledge_qa 三类直接跳过,不产出任何候选事实。
func isSkipIntent(intent string) bool {
switch strings.ToLower(strings.TrimSpace(intent)) {
case "chitchat", "task_request", "knowledge_qa":
return true
default:
return false
}
}
func truncateForLog(raw *llmservice.TextResult) string {
if raw == nil {
return ""
}
text := strings.TrimSpace(raw.Text)
if len(text) <= 200 {
return text
}
return text[:200] + "..."
}

View File

@@ -0,0 +1,40 @@
package orchestrator
import (
"context"
"strings"
memoryutils "github.com/LoveLosita/smartflow/backend/services/memory/internal/utils"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
// WriteOrchestrator 是 Day1 的本地回退版本。
//
// 职责边界:
// 1. 只做最保守的“从 source_text 直接生成一条候选事实”;
// 2. 不依赖 LLM便于在模型不可用时保底
// 3. 后续会逐步被 LLM 版编排器取代,但不会直接删掉,方便回退。
type WriteOrchestrator struct{}
func NewWriteOrchestrator() *WriteOrchestrator {
return &WriteOrchestrator{}
}
// ExtractFacts 执行最小回退链路。
func (o *WriteOrchestrator) ExtractFacts(_ context.Context, payload memorymodel.ExtractJobPayload) ([]memorymodel.NormalizedFact, error) {
sourceText := strings.TrimSpace(payload.SourceText)
if sourceText == "" {
return nil, nil
}
candidates := []memorymodel.FactCandidate{{
MemoryType: memorymodel.MemoryTypeFact,
Title: "用户提到",
Content: sourceText,
Confidence: 0.6,
Importance: 0.6,
SensitivityLevel: 0,
IsExplicit: false,
}}
return memoryutils.NormalizeFacts(candidates), nil
}

View File

@@ -0,0 +1,29 @@
package repo
import (
"context"
"errors"
"github.com/LoveLosita/smartflow/backend/model"
"gorm.io/gorm"
)
// AuditRepo 封装 memory_audit_logs 的数据访问。
type AuditRepo struct {
db *gorm.DB
}
func NewAuditRepo(db *gorm.DB) *AuditRepo {
return &AuditRepo{db: db}
}
func (r *AuditRepo) WithTx(tx *gorm.DB) *AuditRepo {
return &AuditRepo{db: tx}
}
func (r *AuditRepo) Create(ctx context.Context, log model.MemoryAuditLog) error {
if r == nil || r.db == nil {
return errors.New("memory audit repo is nil")
}
return r.db.WithContext(ctx).Create(&log).Error
}

View File

@@ -0,0 +1,568 @@
package repo
import (
"context"
"errors"
"strings"
"time"
"github.com/LoveLosita/smartflow/backend/model"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
"gorm.io/gorm"
)
// ItemRepo 封装 memory_items 的数据访问。
//
// 职责边界:
// 1. 只负责表级读写,不承载注入、重排、审计决策;
// 2. 查询条件统一由 ItemQuery 表达,避免 service 层拼装 SQL
// 3. 软删除、访问时间刷新等状态变更也收敛到这里。
type ItemRepo struct {
db *gorm.DB
}
func NewItemRepo(db *gorm.DB) *ItemRepo {
return &ItemRepo{db: db}
}
func (r *ItemRepo) WithTx(tx *gorm.DB) *ItemRepo {
return &ItemRepo{db: tx}
}
// UpsertItems 批量写入记忆条目。
func (r *ItemRepo) UpsertItems(ctx context.Context, items []model.MemoryItem) error {
if r == nil || r.db == nil {
return errors.New("memory item repo is nil")
}
if len(items) == 0 {
return nil
}
for i := range items {
if err := r.db.WithContext(ctx).Create(&items[i]).Error; err != nil {
return err
}
}
return nil
}
// Create 写入单条记忆并返回带自增主键的结果。
//
// 职责边界:
// 1. 只负责单条落库,不负责内容归一化与业务校验;
// 2. 默认把 vector_status 视为上游已决策好的桥接状态,不在这里擅自改写;
// 3. 返回值用于上游继续写 audit 或做向量同步。
func (r *ItemRepo) Create(ctx context.Context, fields memorymodel.CreateItemFields) (*model.MemoryItem, error) {
if r == nil || r.db == nil {
return nil, errors.New("memory item repo is nil")
}
if fields.UserID <= 0 {
return nil, errors.New("memory item create user_id is invalid")
}
item := model.MemoryItem{
UserID: fields.UserID,
ConversationID: strPtrOrNil(fields.ConversationID),
AssistantID: strPtrOrNil(fields.AssistantID),
RunID: strPtrOrNil(fields.RunID),
MemoryType: fields.MemoryType,
Title: fields.Title,
Content: fields.Content,
NormalizedContent: strPtrOrNil(fields.NormalizedContent),
ContentHash: strPtrOrNil(fields.ContentHash),
Confidence: fields.Confidence,
Importance: fields.Importance,
SensitivityLevel: fields.SensitivityLevel,
SourceMessageID: fields.SourceMessageID,
SourceEventID: fields.SourceEventID,
IsExplicit: fields.IsExplicit,
Status: fields.Status,
TTLAt: fields.TTLAt,
LastAccessAt: fields.LastAccessAt,
VectorStatus: fields.VectorStatus,
}
if item.Status == "" {
item.Status = model.MemoryItemStatusActive
}
if strings.TrimSpace(item.VectorStatus) == "" {
item.VectorStatus = "pending"
}
if err := r.db.WithContext(ctx).Create(&item).Error; err != nil {
return nil, err
}
return &item, nil
}
// FindByQuery 按统一过滤条件读取记忆条目。
//
// 步骤化说明:
// 1. 先强制 user_id 过滤,避免跨用户串记忆;
// 2. 再按会话/助手/run 维度补充过滤IncludeGlobal=true 时允许读取对应全局条目;
// 3. 最后补状态、类型、过期时间和 limit返回稳定排序结果。
func (r *ItemRepo) FindByQuery(ctx context.Context, query memorymodel.ItemQuery) ([]model.MemoryItem, error) {
if r == nil || r.db == nil {
return nil, errors.New("memory item repo is nil")
}
if query.UserID <= 0 {
return nil, errors.New("memory item query user_id is invalid")
}
db := r.db.WithContext(ctx).Model(&model.MemoryItem{}).Where("user_id = ?", query.UserID)
db = applyScopedEquality(db, "conversation_id", query.ConversationID, query.IncludeGlobal)
db = applyScopedEquality(db, "assistant_id", query.AssistantID, query.IncludeGlobal)
db = applyScopedEquality(db, "run_id", query.RunID, query.IncludeGlobal)
if len(query.Statuses) > 0 {
db = db.Where("status IN ?", query.Statuses)
}
if len(query.MemoryTypes) > 0 {
db = db.Where("memory_type IN ?", query.MemoryTypes)
}
if query.OnlyUnexpired {
now := query.Now
if now.IsZero() {
now = time.Now()
}
db = db.Where("(ttl_at IS NULL OR ttl_at > ?)", now)
}
if query.Limit > 0 {
db = db.Limit(query.Limit)
}
var items []model.MemoryItem
err := db.
Order("is_explicit DESC").
Order("importance DESC").
Order("updated_at DESC").
Find(&items).Error
return items, err
}
// FindPinnedByUser 读取“应优先注入”的结构化记忆。
//
// 步骤化说明:
// 1. 先在同一组 user/conversation/assistant/run 作用域下查 constraint保证硬约束不会因语义召回波动丢失
// 2. 再查高置信 preference并按 importance 降序裁到预算,避免偏好噪声过多;
// 3. 两路结果按“constraint 在前、preference 在后”拼接,后续由 service 层统一去重、排序和预算裁剪;
// 4. 这里不直接做最终预算,是因为读取侧还要和语义候选合并后统一重排。
func (r *ItemRepo) FindPinnedByUser(
ctx context.Context,
query memorymodel.ItemQuery,
preferenceLimit int,
) ([]model.MemoryItem, error) {
if r == nil || r.db == nil {
return nil, errors.New("memory item repo is nil")
}
if query.UserID <= 0 {
return nil, errors.New("memory item query user_id is invalid")
}
includeConstraint := allowPinnedMemoryType(query.MemoryTypes, memorymodel.MemoryTypeConstraint)
includePreference := allowPinnedMemoryType(query.MemoryTypes, memorymodel.MemoryTypePreference)
if !includeConstraint && !includePreference {
return nil, nil
}
base := r.db.WithContext(ctx).Model(&model.MemoryItem{}).Where("user_id = ?", query.UserID)
base = applyScopedEquality(base, "conversation_id", query.ConversationID, query.IncludeGlobal)
base = applyScopedEquality(base, "assistant_id", query.AssistantID, query.IncludeGlobal)
base = applyScopedEquality(base, "run_id", query.RunID, query.IncludeGlobal)
base = applyPinnedUnexpiredScope(base, query)
result := make([]model.MemoryItem, 0, preferenceLimit+4)
if includeConstraint {
var constraints []model.MemoryItem
err := base.Session(&gorm.Session{}).
Where("memory_type = ? AND status = ?", memorymodel.MemoryTypeConstraint, model.MemoryItemStatusActive).
Order("importance DESC").
Order("updated_at DESC").
Find(&constraints).Error
if err != nil {
return nil, err
}
result = append(result, constraints...)
}
if includePreference {
if preferenceLimit <= 0 {
preferenceLimit = memorymodel.DefaultReadPreferenceLimit
}
var preferences []model.MemoryItem
err := base.Session(&gorm.Session{}).
Where("memory_type = ? AND confidence >= ? AND status = ?", memorymodel.MemoryTypePreference, 0.8, model.MemoryItemStatusActive).
Order("importance DESC").
Order("updated_at DESC").
Limit(preferenceLimit).
Find(&preferences).Error
if err != nil {
return nil, err
}
result = append(result, preferences...)
}
return result, nil
}
// GetByIDForUser 读取某个用户的一条记忆条目。
func (r *ItemRepo) GetByIDForUser(ctx context.Context, userID int, memoryID int64) (*model.MemoryItem, error) {
if r == nil || r.db == nil {
return nil, errors.New("memory item repo is nil")
}
if userID <= 0 || memoryID <= 0 {
return nil, errors.New("memory item query params is invalid")
}
var item model.MemoryItem
err := r.db.WithContext(ctx).
Where("id = ? AND user_id = ?", memoryID, userID).
First(&item).Error
if err != nil {
return nil, err
}
return &item, nil
}
// UpdateStatusByID 更新某条记忆的状态。
func (r *ItemRepo) UpdateStatusByID(ctx context.Context, userID int, memoryID int64, status string) error {
return r.UpdateStatusByIDAt(ctx, userID, memoryID, status, time.Now())
}
// UpdateStatusByIDAt 更新某条记忆的状态,并允许上层显式指定更新时间。
//
// 这样做的原因:
// 1. 管理侧删除时,需要让“库内更新时间”和“审计 after 快照时间”保持一致;
// 2. 读取侧若只是刷新 last_access_at不应该误改 updated_at
// 3. 因此把“更新时间来源”收口到 repo避免 service 层自己拼 SQL。
func (r *ItemRepo) UpdateStatusByIDAt(
ctx context.Context,
userID int,
memoryID int64,
status string,
updatedAt time.Time,
) error {
if r == nil || r.db == nil {
return errors.New("memory item repo is nil")
}
if userID <= 0 || memoryID <= 0 {
return errors.New("memory item update params is invalid")
}
status = strings.TrimSpace(status)
if status == "" {
return errors.New("memory item status is empty")
}
if updatedAt.IsZero() {
updatedAt = time.Now()
}
return r.db.WithContext(ctx).
Model(&model.MemoryItem{UserID: userID}).
Where("id = ? AND user_id = ?", memoryID, userID).
Updates(map[string]any{
"status": status,
"updated_at": updatedAt,
}).Error
}
// TouchLastAccessAt 批量刷新记忆访问时间。
//
// 说明:
// 1. 这里只更新 last_access_at不更新 updated_at
// 2. 因为 updated_at 代表“内容被修改”的时间,不能被一次普通读取污染;
// 3. 否则后续读取重排会把“最近被读过的旧记忆”误判成“最近被更新的记忆”。
func (r *ItemRepo) TouchLastAccessAt(ctx context.Context, ids []int64, accessedAt time.Time) error {
if r == nil || r.db == nil {
return errors.New("memory item repo is nil")
}
if len(ids) == 0 {
return nil
}
if accessedAt.IsZero() {
accessedAt = time.Now()
}
return r.db.WithContext(ctx).
Model(&model.MemoryItem{}).
Where("id IN ?", ids).
Updates(map[string]any{
"last_access_at": accessedAt,
}).Error
}
// UpdateVectorStateByID 更新单条记忆的向量同步桥接状态。
//
// 说明:
// 1. 这里只更新 vector_status/vector_id不更新 updated_at
// 2. 因为向量同步属于索引层状态,不代表记忆内容本身被修改;
// 3. 若误改 updated_at会污染读取侧的时间排序语义。
func (r *ItemRepo) UpdateVectorStateByID(
ctx context.Context,
memoryID int64,
vectorStatus string,
vectorID *string,
) error {
if r == nil || r.db == nil {
return errors.New("memory item repo is nil")
}
if memoryID <= 0 {
return errors.New("memory item vector update id is invalid")
}
vectorStatus = strings.TrimSpace(vectorStatus)
if vectorStatus == "" {
return errors.New("memory item vector status is empty")
}
return r.db.WithContext(ctx).
Model(&model.MemoryItem{}).
Where("id = ?", memoryID).
UpdateColumns(map[string]any{
"vector_status": vectorStatus,
"vector_id": vectorID,
}).Error
}
// FindActiveByHash 按用户和内容哈希精确查找活跃记忆。
//
// 用途:
// 1. 决策层 Step 1 的 Hash 精确命中检查;
// 2. 利用 idx_memory_items_user_type_hash 联合索引,避免全表扫描;
// 3. 只返回 status=active 的记录,软删除记录不参与去重。
func (r *ItemRepo) FindActiveByHash(ctx context.Context, userID int, contentHash string) ([]model.MemoryItem, error) {
if r == nil || r.db == nil {
return nil, errors.New("memory item repo is nil")
}
if userID <= 0 || strings.TrimSpace(contentHash) == "" {
return nil, errors.New("memory item find by hash params is invalid")
}
var items []model.MemoryItem
err := r.db.WithContext(ctx).
Where("user_id = ? AND content_hash = ? AND status = ?", userID, contentHash, model.MemoryItemStatusActive).
Find(&items).Error
return items, err
}
// UpdateContentByID 更新指定记忆的内容相关字段。
//
// 步骤化说明:
// 1. 只改 title/content/normalized_content/content_hash/confidence/importance 六个字段;
// 2. 不改 status/user_id/memory_type 等身份字段,保证更新操作不改变记忆归属;
// 3. updated_at 由 GORM AutoUpdateTime 自动维护。
func (r *ItemRepo) UpdateContentByID(ctx context.Context, memoryID int64, fields memorymodel.UpdateContentFields) error {
if r == nil || r.db == nil {
return errors.New("memory item repo is nil")
}
if memoryID <= 0 {
return errors.New("memory item update content id is invalid")
}
return r.db.WithContext(ctx).
Model(&model.MemoryItem{}).
Where("id = ?", memoryID).
Updates(map[string]any{
"title": fields.Title,
"content": fields.Content,
"normalized_content": fields.NormalizedContent,
"content_hash": fields.ContentHash,
"confidence": fields.Confidence,
"importance": fields.Importance,
}).Error
}
// UpdateManagedFieldsByID 更新“用户管理侧”允许修改的记忆字段。
func (r *ItemRepo) UpdateManagedFieldsByID(ctx context.Context, userID int, memoryID int64, fields memorymodel.UpdateItemFields) error {
return r.UpdateManagedFieldsByIDAt(ctx, userID, memoryID, fields, time.Now())
}
// UpdateManagedFieldsByIDAt 更新“用户管理侧”允许修改的记忆字段,并允许显式指定更新时间。
//
// 步骤化说明:
// 1. 这里只改内容侧和展示侧字段,不改 user_id/status 等归属语义;
// 2. memory_type/content 变化后,会把 vector_status 置为 pending提示上游需要重新同步向量
// 3. TTLAt 允许被设置为 nil用于显式清空过期时间。
func (r *ItemRepo) UpdateManagedFieldsByIDAt(
ctx context.Context,
userID int,
memoryID int64,
fields memorymodel.UpdateItemFields,
updatedAt time.Time,
) error {
if r == nil || r.db == nil {
return errors.New("memory item repo is nil")
}
if userID <= 0 || memoryID <= 0 {
return errors.New("memory item update params is invalid")
}
if updatedAt.IsZero() {
updatedAt = time.Now()
}
return r.db.WithContext(ctx).
Model(&model.MemoryItem{UserID: userID}).
Where("id = ? AND user_id = ?", memoryID, userID).
Updates(map[string]any{
"memory_type": fields.MemoryType,
"title": fields.Title,
"content": fields.Content,
"normalized_content": fields.NormalizedContent,
"content_hash": fields.ContentHash,
"confidence": fields.Confidence,
"importance": fields.Importance,
"sensitivity_level": fields.SensitivityLevel,
"is_explicit": fields.IsExplicit,
"ttl_at": fields.TTLAt,
"vector_status": "pending",
"updated_at": updatedAt,
}).Error
}
// SoftDeleteByID 软删除指定用户的某条记忆。
//
// 说明:
// 1. 复用 UpdateStatusByIDAt 的逻辑模式,把 status 改为 deleted
// 2. 同时把 vector_status 重置为 pending确保向量侧也能感知删除
// 3. 必须带 user_id 条件,避免跨用户误删。
func (r *ItemRepo) SoftDeleteByID(ctx context.Context, userID int, memoryID int64) error {
if r == nil || r.db == nil {
return errors.New("memory item repo is nil")
}
if userID <= 0 || memoryID <= 0 {
return errors.New("memory item soft delete params is invalid")
}
return r.db.WithContext(ctx).
Model(&model.MemoryItem{UserID: userID}).
Where("id = ? AND user_id = ?", memoryID, userID).
Updates(map[string]any{
"status": model.MemoryItemStatusDeleted,
"vector_status": "pending",
"updated_at": time.Now(),
}).Error
}
// RestoreByID 把 deleted/archived 记忆恢复为 active。
func (r *ItemRepo) RestoreByID(ctx context.Context, userID int, memoryID int64) error {
return r.RestoreByIDAt(ctx, userID, memoryID, time.Now())
}
// RestoreByIDAt 把 deleted/archived 记忆恢复为 active并显式刷新 vector_status。
//
// 这样做的原因:
// 1. 恢复后的记忆需要重新参与语义召回,因此向量侧也要重新同步;
// 2. 这里统一把 vector_status 置为 pending避免上游遗漏桥接状态更新
// 3. 若目标记录本身已是 active上游应先读快照决定是否真的调用恢复。
func (r *ItemRepo) RestoreByIDAt(ctx context.Context, userID int, memoryID int64, updatedAt time.Time) error {
if r == nil || r.db == nil {
return errors.New("memory item repo is nil")
}
if userID <= 0 || memoryID <= 0 {
return errors.New("memory item restore params is invalid")
}
if updatedAt.IsZero() {
updatedAt = time.Now()
}
return r.db.WithContext(ctx).
Model(&model.MemoryItem{UserID: userID}).
Where("id = ? AND user_id = ?", memoryID, userID).
Updates(map[string]any{
"status": model.MemoryItemStatusActive,
"vector_status": "pending",
"updated_at": updatedAt,
}).Error
}
// ArchiveByIDsAt 把一批重复记忆改为 archived并等待上游删除向量副本。
func (r *ItemRepo) ArchiveByIDsAt(ctx context.Context, ids []int64, updatedAt time.Time) error {
if r == nil || r.db == nil {
return errors.New("memory item repo is nil")
}
if len(ids) == 0 {
return nil
}
if updatedAt.IsZero() {
updatedAt = time.Now()
}
return r.db.WithContext(ctx).
Model(&model.MemoryItem{}).
Where("id IN ?", ids).
Where("status = ?", model.MemoryItemStatusActive).
Updates(map[string]any{
"status": model.MemoryItemStatusArchived,
"vector_status": "pending",
"updated_at": updatedAt,
}).Error
}
// ListActiveItemsForDedup 读取“当前仍 active 且带 content_hash”的候选记忆供离线 dedup 治理使用。
//
// 步骤化说明:
// 1. 只扫描 status=active 且 hash 非空的记录,因为治理目标是“活跃重复项”;
// 2. 先按 user/type/hash 分组,再按更新时间、置信度、主键逆序排列,方便上游顺序分组;
// 3. Limit 仅用于保守控量,不保证整组完整,因此首次治理建议留空或给足够大值。
func (r *ItemRepo) ListActiveItemsForDedup(ctx context.Context, userID int, limit int) ([]model.MemoryItem, error) {
if r == nil || r.db == nil {
return nil, errors.New("memory item repo is nil")
}
db := r.db.WithContext(ctx).
Model(&model.MemoryItem{}).
Where("status = ?", model.MemoryItemStatusActive).
Where("content_hash IS NOT NULL AND content_hash <> ''")
if userID > 0 {
db = db.Where("user_id = ?", userID)
}
if limit > 0 {
db = db.Limit(limit)
}
var items []model.MemoryItem
err := db.
Order("user_id ASC").
Order("memory_type ASC").
Order("content_hash ASC").
Order("updated_at DESC").
Order("confidence DESC").
Order("id DESC").
Find(&items).Error
return items, err
}
func applyScopedEquality(db *gorm.DB, column, value string, includeGlobal bool) *gorm.DB {
value = strings.TrimSpace(value)
if value == "" {
return db
}
if includeGlobal {
return db.Where("("+column+" = ? OR "+column+" IS NULL)", value)
}
return db.Where(column+" = ?", value)
}
func applyPinnedUnexpiredScope(db *gorm.DB, query memorymodel.ItemQuery) *gorm.DB {
if db == nil || !query.OnlyUnexpired {
return db
}
now := query.Now
if now.IsZero() {
now = time.Now()
}
return db.Where("(ttl_at IS NULL OR ttl_at > ?)", now)
}
func allowPinnedMemoryType(memoryTypes []string, target string) bool {
if len(memoryTypes) == 0 {
return true
}
target = memorymodel.NormalizeMemoryType(target)
for _, item := range memoryTypes {
if memorymodel.NormalizeMemoryType(item) == target {
return true
}
}
return false
}

View File

@@ -0,0 +1,222 @@
package repo
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/LoveLosita/smartflow/backend/model"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// JobRepo 封装 memory_jobs 的数据访问。
type JobRepo struct {
db *gorm.DB
}
func NewJobRepo(db *gorm.DB) *JobRepo {
return &JobRepo{db: db}
}
func (r *JobRepo) WithTx(tx *gorm.DB) *JobRepo {
return &JobRepo{db: tx}
}
// CreatePendingExtractJob 创建“待抽取”任务(幂等写入)。
//
// 失败语义:
// 1. 参数非法直接返回 error由上游决定 dead 或重试;
// 2. 同幂等键重复写入采用 DoNothing保证无副作用。
func (r *JobRepo) CreatePendingExtractJob(
ctx context.Context,
payload memorymodel.ExtractJobPayload,
sourceEventID string,
) error {
if r == nil || r.db == nil {
return errors.New("memory job repo is nil")
}
if payload.UserID <= 0 {
return errors.New("invalid user_id")
}
if payload.IdempotencyKey == "" {
return errors.New("idempotency_key is empty")
}
rawPayload, err := json.Marshal(payload)
if err != nil {
return err
}
now := time.Now()
job := model.MemoryJob{
UserID: payload.UserID,
ConversationID: strPtrOrNil(payload.ConversationID),
SourceMessageID: int64PtrOrNil(payload.SourceMessageID),
SourceEventID: strPtrOrNil(sourceEventID),
JobType: model.MemoryJobTypeExtract,
IdempotencyKey: payload.IdempotencyKey,
PayloadJSON: string(rawPayload),
Status: model.MemoryJobStatusPending,
RetryCount: 0,
MaxRetry: 6,
NextRetryAt: &now,
}
return r.db.WithContext(ctx).
Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "idempotency_key"}},
DoNothing: true,
}).
Create(&job).Error
}
// ClaimNextRunnableExtractJob 抢占一个可执行的 extract 任务。
//
// 抢占规则:
// 1. 只从 pending/failed 中挑 next_retry_at 已到期任务;
// 2. 用行锁避免多个 worker 抢到同一条任务;
// 3. 抢占成功后立即置为 processing防止重复执行。
func (r *JobRepo) ClaimNextRunnableExtractJob(ctx context.Context, now time.Time) (*model.MemoryJob, error) {
if r == nil || r.db == nil {
return nil, errors.New("memory job repo is nil")
}
var claimed *model.MemoryJob
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var job model.MemoryJob
query := tx.
Clauses(clause.Locking{Strength: "UPDATE"}).
Where("job_type = ?", model.MemoryJobTypeExtract).
Where("status IN ?", []string{model.MemoryJobStatusPending, model.MemoryJobStatusFailed}).
Where("(next_retry_at IS NULL OR next_retry_at <= ?)", now).
Order("id ASC").
Limit(1).
Find(&job)
if query.Error != nil {
return query.Error
}
if query.RowsAffected == 0 {
return nil
}
updates := map[string]any{
"status": model.MemoryJobStatusProcessing,
"updated_at": now,
"last_error": nil,
}
if updateErr := tx.Model(&model.MemoryJob{}).Where("id = ?", job.ID).Updates(updates).Error; updateErr != nil {
return updateErr
}
job.Status = model.MemoryJobStatusProcessing
job.UpdatedAt = &now
claimed = &job
return nil
})
if err != nil {
return nil, err
}
return claimed, nil
}
// MarkSuccess 把任务推进为 success 最终态。
func (r *JobRepo) MarkSuccess(ctx context.Context, jobID int64) error {
if r == nil || r.db == nil {
return errors.New("memory job repo is nil")
}
now := time.Now()
updates := map[string]any{
"status": model.MemoryJobStatusSuccess,
"last_error": nil,
"next_retry_at": nil,
"updated_at": now,
}
return r.db.WithContext(ctx).Model(&model.MemoryJob{}).Where("id = ?", jobID).Updates(updates).Error
}
// MarkFailed 按重试策略推进任务到 failed/dead。
//
// 规则:
// 1. retry_count +1 后若超上限,直接 dead
// 2. 未超上限则写 failed 并设置 next_retry_at。
func (r *JobRepo) MarkFailed(ctx context.Context, jobID int64, reason string) error {
if r == nil || r.db == nil {
return errors.New("memory job repo is nil")
}
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var job model.MemoryJob
queryErr := tx.
Clauses(clause.Locking{Strength: "UPDATE"}).
Where("id = ?", jobID).
First(&job).Error
if queryErr != nil {
return queryErr
}
if job.Status == model.MemoryJobStatusSuccess || job.Status == model.MemoryJobStatusDead {
return nil
}
maxRetry := job.MaxRetry
if maxRetry <= 0 {
maxRetry = 6
}
nextRetryCount := job.RetryCount + 1
now := time.Now()
status := model.MemoryJobStatusFailed
var nextRetryAt *time.Time
if nextRetryCount >= maxRetry {
status = model.MemoryJobStatusDead
nextRetryAt = nil
} else {
t := now.Add(calcRetryBackoff(nextRetryCount))
nextRetryAt = &t
}
lastErr := truncateError(reason)
updates := map[string]any{
"status": status,
"retry_count": nextRetryCount,
"last_error": &lastErr,
"next_retry_at": nextRetryAt,
"updated_at": now,
}
return tx.Model(&model.MemoryJob{}).Where("id = ?", jobID).Updates(updates).Error
})
}
func calcRetryBackoff(retryCount int) time.Duration {
if retryCount <= 0 {
return time.Second
}
if retryCount > 6 {
retryCount = 6
}
return time.Second * time.Duration(1<<(retryCount-1))
}
func truncateError(reason string) string {
if len(reason) <= 2000 {
return reason
}
return reason[:2000]
}
func strPtrOrNil(v string) *string {
if v == "" {
return nil
}
value := v
return &value
}
func int64PtrOrNil(v int64) *int64 {
if v <= 0 {
return nil
}
value := v
return &value
}

View File

@@ -0,0 +1,64 @@
package repo
import (
"context"
"errors"
"github.com/LoveLosita/smartflow/backend/model"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// SettingsRepo 封装 memory_user_settings 的读写。
type SettingsRepo struct {
db *gorm.DB
}
func NewSettingsRepo(db *gorm.DB) *SettingsRepo {
return &SettingsRepo{db: db}
}
func (r *SettingsRepo) WithTx(tx *gorm.DB) *SettingsRepo {
return &SettingsRepo{db: tx}
}
// GetByUserID 读取用户记忆设置。
//
// 返回语义:
// 1. 命中时返回真实记录;
// 2. 未命中时返回 nil,nil由上层决定是否走默认开关
// 3. 不在仓储层偷偷补默认值,避免写路径和读路径语义不一致。
func (r *SettingsRepo) GetByUserID(ctx context.Context, userID int) (*model.MemoryUserSetting, error) {
if r == nil || r.db == nil {
return nil, errors.New("memory settings repo is nil")
}
if userID <= 0 {
return nil, errors.New("memory settings user_id is invalid")
}
var setting model.MemoryUserSetting
query := r.db.WithContext(ctx).Where("user_id = ?", userID).Limit(1).Find(&setting)
if query.Error != nil {
return nil, query.Error
}
if query.RowsAffected == 0 {
return nil, nil
}
return &setting, nil
}
// Upsert 写入用户记忆设置。
func (r *SettingsRepo) Upsert(ctx context.Context, setting model.MemoryUserSetting) error {
if r == nil || r.db == nil {
return errors.New("memory settings repo is nil")
}
return r.db.WithContext(ctx).Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "user_id"}},
DoUpdates: clause.AssignmentColumns([]string{
"memory_enabled",
"implicit_memory_enabled",
"sensitive_memory_enabled",
"updated_at",
}),
}).Create(&setting).Error
}

View File

@@ -0,0 +1,149 @@
package service
import (
"strings"
"github.com/LoveLosita/smartflow/backend/model"
memoryutils "github.com/LoveLosita/smartflow/backend/services/memory/internal/utils"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
func toItemDTO(item model.MemoryItem) memorymodel.ItemDTO {
return memorymodel.ItemDTO{
ID: item.ID,
UserID: item.UserID,
ConversationID: strValue(item.ConversationID),
AssistantID: strValue(item.AssistantID),
RunID: strValue(item.RunID),
MemoryType: item.MemoryType,
Title: item.Title,
Content: item.Content,
ContentHash: fallbackContentHash(item.MemoryType, item.Content, strValue(item.ContentHash)),
Confidence: item.Confidence,
Importance: item.Importance,
SensitivityLevel: item.SensitivityLevel,
IsExplicit: item.IsExplicit,
Status: item.Status,
TTLAt: item.TTLAt,
CreatedAt: item.CreatedAt,
UpdatedAt: item.UpdatedAt,
}
}
func toItemDTOs(items []model.MemoryItem) []memorymodel.ItemDTO {
if len(items) == 0 {
return nil
}
result := make([]memorymodel.ItemDTO, 0, len(items))
for _, item := range items {
result = append(result, toItemDTO(item))
}
return result
}
func toUserSettingDTO(setting model.MemoryUserSetting) memorymodel.UserSettingDTO {
return memorymodel.UserSettingDTO{
UserID: setting.UserID,
MemoryEnabled: setting.MemoryEnabled,
ImplicitMemoryEnabled: setting.ImplicitMemoryEnabled,
SensitiveMemoryEnabled: setting.SensitiveMemoryEnabled,
UpdatedAt: setting.UpdatedAt,
}
}
func normalizeMemoryTypes(raw []string) []string {
if len(raw) == 0 {
return nil
}
result := make([]string, 0, len(raw))
seen := make(map[string]struct{}, len(raw))
for _, item := range raw {
normalized := memorymodel.NormalizeMemoryType(item)
if normalized == "" {
continue
}
if _, exists := seen[normalized]; exists {
continue
}
seen[normalized] = struct{}{}
result = append(result, normalized)
}
return result
}
func normalizeManageStatuses(raw []string) []string {
if len(raw) == 0 {
return []string{
model.MemoryItemStatusActive,
model.MemoryItemStatusArchived,
}
}
result := make([]string, 0, len(raw))
seen := make(map[string]struct{}, len(raw))
for _, item := range raw {
status := strings.ToLower(strings.TrimSpace(item))
if status != model.MemoryItemStatusActive &&
status != model.MemoryItemStatusArchived &&
status != model.MemoryItemStatusDeleted {
continue
}
if _, exists := seen[status]; exists {
continue
}
seen[status] = struct{}{}
result = append(result, status)
}
if len(result) == 0 {
return []string{
model.MemoryItemStatusActive,
model.MemoryItemStatusArchived,
}
}
return result
}
func normalizeLimit(limit, defaultValue, maxValue int) int {
if limit <= 0 {
limit = defaultValue
}
if maxValue > 0 && limit > maxValue {
return maxValue
}
return limit
}
func strValue(v *string) string {
if v == nil {
return ""
}
return strings.TrimSpace(*v)
}
// fallbackContentHash 返回条目可用于服务级去重的内容哈希。
//
// 说明:
// 1. 优先复用库内已落表的 content_hash避免同一条数据多套算法口径不一致
// 2. 若历史数据或 RAG metadata 没带 hash则按“类型 + 规范化内容”补算;
// 3. 若类型非法或正文为空,则返回空字符串,让上游继续走文本兜底去重。
func fallbackContentHash(memoryType, content, currentHash string) string {
currentHash = strings.TrimSpace(currentHash)
if currentHash != "" {
return currentHash
}
normalizedType := memorymodel.NormalizeMemoryType(memoryType)
normalizedContent := normalizeContentForHash(content)
if normalizedType == "" || normalizedContent == "" {
return ""
}
return memoryutils.HashContent(normalizedType, normalizedContent)
}
func normalizeContentForHash(content string) string {
content = strings.TrimSpace(content)
if content == "" {
return ""
}
return strings.ToLower(strings.Join(strings.Fields(content), " "))
}

View File

@@ -0,0 +1,91 @@
package service
import (
"time"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
"github.com/spf13/viper"
)
// LoadConfigFromViper 读取记忆模块配置并做默认值兜底。
//
// 默认策略:
// 1. temperature/top_p 使用低随机参数,提升可复现性;
// 2. Day1 先提供参数位,不强制所有参数立即生效;
// 3. 轮询与重试参数给出保守默认值,避免对主链路造成压力。
func LoadConfigFromViper() memorymodel.Config {
cfg := memorymodel.Config{
Enabled: viper.GetBool("memory.enabled"),
RAGEnabled: viper.GetBool("memory.rag.enabled"),
ReadMode: memorymodel.NormalizeReadMode(viper.GetString("memory.read.mode")),
InjectRenderMode: memorymodel.NormalizeInjectRenderMode(viper.GetString("memory.inject.renderMode")),
ExtractPrompt: viper.GetString("memory.prompt.extract"),
DecisionPrompt: viper.GetString("memory.prompt.decision"),
Threshold: viper.GetFloat64("memory.threshold"),
EnableReranker: viper.GetBool("memory.enableReranker"),
LLMTemperature: viper.GetFloat64("memory.llm.temperature"),
LLMTopP: viper.GetFloat64("memory.llm.topP"),
JobMaxRetry: viper.GetInt("memory.job.maxRetry"),
WorkerPollEvery: viper.GetDuration("memory.worker.pollEvery"),
WorkerClaimBatch: viper.GetInt("memory.worker.claimBatch"),
ReadConstraintLimit: viper.GetInt("memory.read.constraintLimit"),
ReadPreferenceLimit: viper.GetInt("memory.read.preferenceLimit"),
ReadFactLimit: viper.GetInt("memory.read.factLimit"),
// 决策层配置:默认关闭,灰度开启后才会生效。
DecisionEnabled: viper.GetBool("memory.decision.enabled"),
DecisionCandidateTopK: viper.GetInt("memory.decision.candidateTopK"),
DecisionCandidateMinScore: viper.GetFloat64("memory.decision.candidateMinScore"),
DecisionFallbackMode: viper.GetString("memory.decision.fallbackMode"),
WriteMode: viper.GetString("memory.write.mode"),
WriteMinConfidence: viper.GetFloat64("memory.write.minConfidence"),
LLMThinking: viper.GetBool("agent.thinking.memory"),
}
if cfg.Threshold <= 0 {
cfg.Threshold = 0.55
}
if cfg.LLMTemperature <= 0 {
cfg.LLMTemperature = 0.1
}
if cfg.LLMTopP <= 0 {
cfg.LLMTopP = 0.2
}
if cfg.JobMaxRetry <= 0 {
cfg.JobMaxRetry = 6
}
if cfg.WorkerPollEvery <= 0 {
cfg.WorkerPollEvery = 2 * time.Second
}
if cfg.WorkerClaimBatch <= 0 {
cfg.WorkerClaimBatch = 1
}
cfg.ReadConstraintLimit = cfg.EffectiveReadConstraintLimit()
cfg.ReadPreferenceLimit = cfg.EffectiveReadPreferenceLimit()
cfg.ReadFactLimit = cfg.EffectiveReadFactLimit()
cfg.ReadMode = cfg.EffectiveReadMode()
cfg.InjectRenderMode = cfg.EffectiveInjectRenderMode()
// 决策层配置默认值兜底。
// 说明:
// 1. TopK 和 MinScore 是 Milvus 召回参数,需要保守默认值避免召回过多噪声候选;
// 2. FallbackMode 默认退回旧路径新增,保证决策流程异常时不丢数据;
// 3. WriteMode 由 DecisionEnabled 隐式决定,这里不做强制联动。
if cfg.DecisionCandidateTopK <= 0 {
cfg.DecisionCandidateTopK = 5
}
if cfg.DecisionCandidateMinScore <= 0 {
cfg.DecisionCandidateMinScore = 0.6
}
if cfg.DecisionFallbackMode == "" {
cfg.DecisionFallbackMode = "legacy_add"
}
if cfg.WriteMode == "" {
cfg.WriteMode = "legacy"
}
if cfg.WriteMinConfidence <= 0 {
cfg.WriteMinConfidence = 0.5
}
return cfg
}

View File

@@ -0,0 +1,33 @@
package service
import (
"context"
"errors"
memoryrepo "github.com/LoveLosita/smartflow/backend/services/memory/internal/repo"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
// EnqueueService 是 Day1 的“任务入队门面”。
//
// 职责边界:
// 1. 只负责把抽取请求入 memory_jobs
// 2. 不负责执行抽取、不负责写 memory_items。
type EnqueueService struct {
jobRepo *memoryrepo.JobRepo
}
func NewEnqueueService(jobRepo *memoryrepo.JobRepo) *EnqueueService {
return &EnqueueService{jobRepo: jobRepo}
}
func (s *EnqueueService) EnqueueExtractJob(
ctx context.Context,
payload memorymodel.ExtractJobPayload,
sourceEventID string,
) error {
if s == nil || s.jobRepo == nil {
return errors.New("memory enqueue service is nil")
}
return s.jobRepo.CreatePendingExtractJob(ctx, payload, sourceEventID)
}

View File

@@ -0,0 +1,659 @@
package service
import (
"context"
"errors"
"strings"
"time"
"github.com/LoveLosita/smartflow/backend/model"
"github.com/LoveLosita/smartflow/backend/respond"
memoryrepo "github.com/LoveLosita/smartflow/backend/services/memory/internal/repo"
memoryutils "github.com/LoveLosita/smartflow/backend/services/memory/internal/utils"
memoryvectorsync "github.com/LoveLosita/smartflow/backend/services/memory/internal/vectorsync"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
memoryobserve "github.com/LoveLosita/smartflow/backend/services/memory/observe"
"gorm.io/gorm"
)
const (
defaultManageListLimit = 20
maxManageListLimit = 100
defaultManualConfidence = 0.95
defaultManualImportance = 0.90
)
// ManageService 负责 memory 模块内部的管理面能力。
//
// 职责边界:
// 1. 负责“列出记忆 / 删除记忆 / 读取与更新用户开关”这类维护动作;
// 2. 负责把用户主动管理行为补充进 memory_audit_logs
// 3. 不负责 prompt 注入、不负责向量召回,也不负责后台抽取任务执行。
type ManageService struct {
db *gorm.DB
itemRepo *memoryrepo.ItemRepo
auditRepo *memoryrepo.AuditRepo
settingsRepo *memoryrepo.SettingsRepo
vectorSyncer *memoryvectorsync.Syncer
observer memoryobserve.Observer
metrics memoryobserve.MetricsRecorder
}
func NewManageService(
db *gorm.DB,
itemRepo *memoryrepo.ItemRepo,
auditRepo *memoryrepo.AuditRepo,
settingsRepo *memoryrepo.SettingsRepo,
vectorSyncer *memoryvectorsync.Syncer,
observer memoryobserve.Observer,
metrics memoryobserve.MetricsRecorder,
) *ManageService {
if observer == nil {
observer = memoryobserve.NewNopObserver()
}
if metrics == nil {
metrics = memoryobserve.NewNopMetrics()
}
return &ManageService{
db: db,
itemRepo: itemRepo,
auditRepo: auditRepo,
settingsRepo: settingsRepo,
vectorSyncer: vectorSyncer,
observer: observer,
metrics: metrics,
}
}
// ListItems 列出某个用户当前可管理的记忆条目。
//
// 说明:
// 1. 这里面向“管理视角”,不会按用户开关再做二次过滤;
// 2. 即便用户暂时关闭 memory总览页仍需要看见已有记忆便于手动删除或核对
// 3. 默认只返回 active/archived除非显式传入 deleted。
func (s *ManageService) ListItems(ctx context.Context, req memorymodel.ListItemsRequest) ([]memorymodel.ItemDTO, error) {
if s == nil || s.itemRepo == nil {
return nil, errors.New("memory manage service is nil")
}
if req.UserID <= 0 {
return nil, nil
}
conversationID := strings.TrimSpace(req.ConversationID)
query := memorymodel.ItemQuery{
UserID: req.UserID,
ConversationID: conversationID,
Statuses: normalizeManageStatuses(req.Statuses),
MemoryTypes: normalizeMemoryTypes(req.MemoryTypes),
IncludeGlobal: conversationID != "",
OnlyUnexpired: false,
Limit: normalizeLimit(req.Limit, defaultManageListLimit, maxManageListLimit),
}
items, err := s.itemRepo.FindByQuery(ctx, query)
if err != nil {
return nil, err
}
return toItemDTOs(items), nil
}
// GetItem 返回“当前用户自己的某条记忆”详情。
func (s *ManageService) GetItem(ctx context.Context, req model.MemoryGetItemRequest) (*memorymodel.ItemDTO, error) {
if s == nil || s.itemRepo == nil {
return nil, errors.New("memory manage service is nil")
}
if req.UserID <= 0 {
return nil, respond.WrongUserID
}
if req.MemoryID <= 0 {
return nil, respond.WrongParamType
}
item, err := s.itemRepo.GetByIDForUser(ctx, req.UserID, req.MemoryID)
if err != nil {
return nil, translateManageError(err)
}
dto := toItemDTO(*item)
return &dto, nil
}
// CreateItem 手动新增一条用户记忆,并补审计与向量同步桥接。
func (s *ManageService) CreateItem(ctx context.Context, req model.MemoryCreateItemRequest) (*memorymodel.ItemDTO, error) {
if s == nil || s.db == nil || s.itemRepo == nil || s.auditRepo == nil {
return nil, errors.New("memory manage service is not initialized")
}
if req.UserID <= 0 {
return nil, respond.WrongUserID
}
fields, err := buildCreateItemFields(req)
if err != nil {
s.recordManageAction(ctx, "create", req.UserID, 0, fields.MemoryType, false, err)
return nil, err
}
var createdItem model.MemoryItem
err = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
itemRepo := s.itemRepo.WithTx(tx)
auditRepo := s.auditRepo.WithTx(tx)
created, createErr := itemRepo.Create(ctx, fields)
if createErr != nil {
return createErr
}
createdItem = *created
audit := memoryutils.BuildItemAuditLog(
createdItem.ID,
createdItem.UserID,
memoryutils.AuditOperationCreate,
memoryutils.NormalizeOperatorType(req.OperatorType),
normalizeManageReason(req.Reason, "用户手动新增记忆"),
nil,
&createdItem,
)
return auditRepo.Create(ctx, audit)
})
if err != nil {
err = translateManageError(err)
s.recordManageAction(ctx, "create", req.UserID, 0, fields.MemoryType, false, err)
return nil, err
}
s.vectorSyncer.Upsert(ctx, "", []model.MemoryItem{createdItem})
s.recordManageAction(ctx, "create", req.UserID, createdItem.ID, createdItem.MemoryType, true, nil)
dto := toItemDTO(createdItem)
return &dto, nil
}
// UpdateItem 手动修改一条用户记忆,并补审计与向量重同步桥接。
func (s *ManageService) UpdateItem(ctx context.Context, req model.MemoryUpdateItemRequest) (*memorymodel.ItemDTO, error) {
if s == nil || s.db == nil || s.itemRepo == nil || s.auditRepo == nil {
return nil, errors.New("memory manage service is not initialized")
}
if req.UserID <= 0 {
return nil, respond.WrongUserID
}
if req.MemoryID <= 0 {
return nil, respond.WrongParamType
}
var updatedItem model.MemoryItem
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
itemRepo := s.itemRepo.WithTx(tx)
auditRepo := s.auditRepo.WithTx(tx)
current, getErr := itemRepo.GetByIDForUser(ctx, req.UserID, req.MemoryID)
if getErr != nil {
return getErr
}
fields, afterItem, buildErr := buildUpdateItemFields(req, *current)
if buildErr != nil {
return buildErr
}
now := time.Now()
afterItem.UpdatedAt = &now
afterItem.VectorStatus = "pending"
if updateErr := itemRepo.UpdateManagedFieldsByIDAt(ctx, req.UserID, req.MemoryID, fields, now); updateErr != nil {
return updateErr
}
audit := memoryutils.BuildItemAuditLog(
current.ID,
current.UserID,
memoryutils.AuditOperationUpdate,
memoryutils.NormalizeOperatorType(req.OperatorType),
normalizeManageReason(req.Reason, "用户手动修改记忆"),
current,
&afterItem,
)
if auditErr := auditRepo.Create(ctx, audit); auditErr != nil {
return auditErr
}
updatedItem = afterItem
return nil
})
if err != nil {
err = translateManageError(err)
s.recordManageAction(ctx, "update", req.UserID, req.MemoryID, resolveUpdateMemoryType(req), false, err)
return nil, err
}
s.vectorSyncer.Upsert(ctx, "", []model.MemoryItem{updatedItem})
s.recordManageAction(ctx, "update", req.UserID, updatedItem.ID, updatedItem.MemoryType, true, nil)
dto := toItemDTO(updatedItem)
return &dto, nil
}
// DeleteItem 软删除一条记忆,并补写审计日志。
//
// 步骤化说明:
// 1. 先在事务里读取当前条目快照,确保审计前镜像和实际删除对象一致;
// 2. 若该条目已是 deleted则直接按幂等语义返回避免重复写多条删除审计
// 3. 状态更新成功后再写 audit log保证“有删除就有审计”失败时整笔事务回滚。
func (s *ManageService) DeleteItem(ctx context.Context, req model.MemoryDeleteItemRequest) (*memorymodel.ItemDTO, error) {
if s == nil || s.db == nil || s.itemRepo == nil || s.auditRepo == nil {
return nil, errors.New("memory manage service is not initialized")
}
if req.UserID <= 0 {
return nil, respond.WrongUserID
}
if req.MemoryID <= 0 {
return nil, respond.WrongParamType
}
now := time.Now()
operatorType := memoryutils.NormalizeOperatorType(req.OperatorType)
reason := normalizeDeleteReason(req.Reason)
var deletedItem model.MemoryItem
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
itemRepo := s.itemRepo.WithTx(tx)
auditRepo := s.auditRepo.WithTx(tx)
current, err := itemRepo.GetByIDForUser(ctx, req.UserID, req.MemoryID)
if err != nil {
return err
}
if current.Status == model.MemoryItemStatusDeleted {
deletedItem = *current
return nil
}
before := *current
after := before
after.Status = model.MemoryItemStatusDeleted
after.UpdatedAt = &now
after.VectorStatus = "pending"
if err = itemRepo.SoftDeleteByID(ctx, req.UserID, req.MemoryID); err != nil {
return err
}
audit := memoryutils.BuildItemAuditLog(
req.MemoryID,
req.UserID,
memoryutils.AuditOperationDelete,
operatorType,
reason,
&before,
&after,
)
if err = auditRepo.Create(ctx, audit); err != nil {
return err
}
deletedItem = after
return nil
})
if err != nil {
err = translateManageError(err)
s.recordManageAction(ctx, "delete", req.UserID, req.MemoryID, "", false, err)
return nil, err
}
if deletedItem.ID <= 0 {
return nil, nil
}
if deletedItem.Status == model.MemoryItemStatusDeleted {
s.vectorSyncer.Delete(ctx, "", []int64{deletedItem.ID})
}
s.recordManageAction(ctx, "delete", req.UserID, deletedItem.ID, deletedItem.MemoryType, true, nil)
result := toItemDTO(deletedItem)
return &result, nil
}
// RestoreItem 把 archived/deleted 记忆恢复为 active并补审计与向量同步桥接。
func (s *ManageService) RestoreItem(ctx context.Context, req model.MemoryRestoreItemRequest) (*memorymodel.ItemDTO, error) {
if s == nil || s.db == nil || s.itemRepo == nil || s.auditRepo == nil {
return nil, errors.New("memory manage service is not initialized")
}
if req.UserID <= 0 {
return nil, respond.WrongUserID
}
if req.MemoryID <= 0 {
return nil, respond.WrongParamType
}
var restoredItem model.MemoryItem
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
itemRepo := s.itemRepo.WithTx(tx)
auditRepo := s.auditRepo.WithTx(tx)
current, getErr := itemRepo.GetByIDForUser(ctx, req.UserID, req.MemoryID)
if getErr != nil {
return getErr
}
if current.Status == model.MemoryItemStatusActive {
restoredItem = *current
return nil
}
now := time.Now()
before := *current
after := before
after.Status = model.MemoryItemStatusActive
after.UpdatedAt = &now
after.VectorStatus = "pending"
if restoreErr := itemRepo.RestoreByIDAt(ctx, req.UserID, req.MemoryID, now); restoreErr != nil {
return restoreErr
}
audit := memoryutils.BuildItemAuditLog(
before.ID,
before.UserID,
memoryutils.AuditOperationRestore,
memoryutils.NormalizeOperatorType(req.OperatorType),
normalizeManageReason(req.Reason, "用户恢复记忆"),
&before,
&after,
)
if auditErr := auditRepo.Create(ctx, audit); auditErr != nil {
return auditErr
}
restoredItem = after
return nil
})
if err != nil {
err = translateManageError(err)
s.recordManageAction(ctx, "restore", req.UserID, req.MemoryID, "", false, err)
return nil, err
}
s.vectorSyncer.Upsert(ctx, "", []model.MemoryItem{restoredItem})
s.recordManageAction(ctx, "restore", req.UserID, restoredItem.ID, restoredItem.MemoryType, true, nil)
dto := toItemDTO(restoredItem)
return &dto, nil
}
// GetUserSetting 返回用户当前生效的记忆开关。
//
// 返回语义:
// 1. 若数据库中还没有记录,返回系统默认开关,而不是 nil
// 2. 这样前端/上层调用方始终拿到完整结构,避免再做一层判空补默认值;
// 3. 这里只读 settings不附带修改动作。
func (s *ManageService) GetUserSetting(ctx context.Context, userID int) (memorymodel.UserSettingDTO, error) {
if s == nil || s.settingsRepo == nil {
return memorymodel.UserSettingDTO{}, errors.New("memory manage service is nil")
}
if userID <= 0 {
return memorymodel.UserSettingDTO{}, nil
}
setting, err := s.settingsRepo.GetByUserID(ctx, userID)
if err != nil {
return memorymodel.UserSettingDTO{}, err
}
return toUserSettingDTO(memoryutils.EffectiveUserSetting(setting, userID)), nil
}
// UpsertUserSetting 写入用户记忆开关。
//
// 说明:
// 1. 当前阶段先直接覆盖三类开关,不做 patch 语义;
// 2. 这样便于前端把整块设置表单一次性提交,接口语义更稳定;
// 3. 若后续需要记录设置变更审计,再单独扩展 setting audit而不是复用 item audit。
func (s *ManageService) UpsertUserSetting(ctx context.Context, req memorymodel.UpdateUserSettingRequest) (memorymodel.UserSettingDTO, error) {
if s == nil || s.settingsRepo == nil {
return memorymodel.UserSettingDTO{}, errors.New("memory manage service is nil")
}
if req.UserID <= 0 {
return memorymodel.UserSettingDTO{}, nil
}
now := time.Now()
setting := model.MemoryUserSetting{
UserID: req.UserID,
MemoryEnabled: req.MemoryEnabled,
ImplicitMemoryEnabled: req.ImplicitMemoryEnabled,
SensitiveMemoryEnabled: req.SensitiveMemoryEnabled,
UpdatedAt: &now,
}
if err := s.settingsRepo.Upsert(ctx, setting); err != nil {
return memorymodel.UserSettingDTO{}, err
}
return toUserSettingDTO(setting), nil
}
func normalizeDeleteReason(reason string) string {
reason = strings.TrimSpace(reason)
if reason == "" {
return "用户删除记忆"
}
return reason
}
func normalizeManageReason(reason string, fallback string) string {
reason = strings.TrimSpace(reason)
if reason == "" {
return fallback
}
return reason
}
func translateManageError(err error) error {
switch {
case err == nil:
return nil
case errors.Is(err, gorm.ErrRecordNotFound):
return respond.MemoryItemNotFound
default:
return err
}
}
func buildCreateItemFields(req model.MemoryCreateItemRequest) (memorymodel.CreateItemFields, error) {
memoryType, err := normalizeManagedMemoryType(req.MemoryType)
if err != nil {
return memorymodel.CreateItemFields{}, err
}
content, normalizedContent, err := normalizeManagedContent(req.Content)
if err != nil {
return memorymodel.CreateItemFields{}, err
}
title := normalizeManagedTitle(req.Title, content)
return memorymodel.CreateItemFields{
UserID: req.UserID,
ConversationID: strings.TrimSpace(req.ConversationID),
AssistantID: strings.TrimSpace(req.AssistantID),
RunID: strings.TrimSpace(req.RunID),
MemoryType: memoryType,
Title: title,
Content: content,
NormalizedContent: normalizedContent,
ContentHash: memoryutils.HashContent(memoryType, normalizedContent),
Confidence: normalizeManageScore(req.Confidence, defaultManualConfidence),
Importance: normalizeManageScore(req.Importance, defaultManualImportance),
SensitivityLevel: normalizeManageSensitivity(req.SensitivityLevel, 0),
IsExplicit: normalizeManageBool(req.IsExplicit, true),
Status: model.MemoryItemStatusActive,
TTLAt: req.TTLAt,
VectorStatus: "pending",
}, nil
}
func buildUpdateItemFields(
req model.MemoryUpdateItemRequest,
current model.MemoryItem,
) (memorymodel.UpdateItemFields, model.MemoryItem, error) {
memoryType := current.MemoryType
if req.MemoryType != nil {
normalizedType, err := normalizeManagedMemoryType(*req.MemoryType)
if err != nil {
return memorymodel.UpdateItemFields{}, model.MemoryItem{}, err
}
memoryType = normalizedType
}
content := current.Content
if req.Content != nil {
normalizedContentValue, _, err := normalizeManagedContent(*req.Content)
if err != nil {
return memorymodel.UpdateItemFields{}, model.MemoryItem{}, err
}
content = normalizedContentValue
}
normalizedContent := normalizeContentForHash(content)
if normalizedContent == "" {
return memorymodel.UpdateItemFields{}, model.MemoryItem{}, respond.MemoryInvalidContent
}
title := current.Title
if req.Title != nil {
title = normalizeManagedTitle(*req.Title, content)
}
ttlAt := current.TTLAt
if req.ClearTTL {
ttlAt = nil
} else if req.TTLAt != nil {
ttlAt = req.TTLAt
}
fields := memorymodel.UpdateItemFields{
MemoryType: memoryType,
Title: title,
Content: content,
NormalizedContent: normalizedContent,
ContentHash: memoryutils.HashContent(memoryType, normalizedContent),
Confidence: normalizeManageScore(req.Confidence, current.Confidence),
Importance: normalizeManageScore(req.Importance, current.Importance),
SensitivityLevel: normalizeManageSensitivity(req.SensitivityLevel, current.SensitivityLevel),
IsExplicit: normalizeManageBool(req.IsExplicit, current.IsExplicit),
TTLAt: ttlAt,
}
after := current
after.MemoryType = fields.MemoryType
after.Title = fields.Title
after.Content = fields.Content
after.NormalizedContent = strPtr(fields.NormalizedContent)
after.ContentHash = strPtr(fields.ContentHash)
after.Confidence = fields.Confidence
after.Importance = fields.Importance
after.SensitivityLevel = fields.SensitivityLevel
after.IsExplicit = fields.IsExplicit
after.TTLAt = fields.TTLAt
return fields, after, nil
}
func normalizeManagedMemoryType(raw string) (string, error) {
normalized := memorymodel.NormalizeMemoryType(raw)
if normalized == "" {
return "", respond.MemoryInvalidType
}
return normalized, nil
}
func normalizeManagedContent(raw string) (string, string, error) {
content := strings.TrimSpace(raw)
if content == "" {
return "", "", respond.MemoryInvalidContent
}
normalized := normalizeContentForHash(content)
if normalized == "" {
return "", "", respond.MemoryInvalidContent
}
return content, normalized, nil
}
func normalizeManagedTitle(raw string, content string) string {
title := strings.TrimSpace(raw)
if title != "" {
return title
}
content = strings.TrimSpace(content)
if content == "" {
return "未命名记忆"
}
runes := []rune(content)
if len(runes) > 24 {
return string(runes[:24])
}
return content
}
func normalizeManageScore(value *float64, defaultValue float64) float64 {
if value == nil {
return clamp01(defaultValue)
}
return clamp01(*value)
}
func normalizeManageSensitivity(value *int, defaultValue int) int {
if value == nil {
return defaultValue
}
if *value < 0 {
return defaultValue
}
return *value
}
func normalizeManageBool(value *bool, defaultValue bool) bool {
if value == nil {
return defaultValue
}
return *value
}
func resolveUpdateMemoryType(req model.MemoryUpdateItemRequest) string {
if req.MemoryType == nil {
return ""
}
return strings.TrimSpace(*req.MemoryType)
}
func strPtr(value string) *string {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
result := value
return &result
}
func (s *ManageService) recordManageAction(
ctx context.Context,
operation string,
userID int,
memoryID int64,
memoryType string,
success bool,
err error,
) {
if s == nil {
return
}
status := "success"
level := memoryobserve.LevelInfo
if !success || err != nil {
status = "error"
level = memoryobserve.LevelWarn
}
s.metrics.AddCounter(memoryobserve.MetricManageTotal, 1, map[string]string{
"operation": strings.TrimSpace(operation),
"status": status,
})
s.observer.Observe(ctx, memoryobserve.Event{
Level: level,
Component: memoryobserve.ComponentManage,
Operation: memoryobserve.OperationManage,
Fields: map[string]any{
"user_id": userID,
"memory_id": memoryID,
"action": strings.TrimSpace(operation),
"memory_type": strings.TrimSpace(memoryType),
"success": success && err == nil,
"error": err,
"error_code": memoryobserve.ClassifyError(err),
},
})
}

View File

@@ -0,0 +1,83 @@
package service
import (
"time"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
ragservice "github.com/LoveLosita/smartflow/backend/services/rag"
)
// buildReadScopedItemQuery 构造读侧统一使用的 MySQL 查询条件。
//
// 职责边界:
// 1. 只负责把 RetrieveRequest 映射成“读侧作用域”查询参数;
// 2. 不负责真正查库,也不负责排序、裁剪或注入;
// 3. conversation_id 字段在这里刻意不参与过滤,仅保留在记忆记录元数据里供审计与溯源使用。
//
// 步骤化说明:
// 1. 读侧始终按 user_id 作为硬隔离边界,避免跨用户串记忆。
// 2. assistant_id / run_id 仍允许参与过滤,因为它们表达的是助手实例与执行轮次边界,而不是“是否跨对话召回”的问题。
// 3. conversation_id 明确置空,原因是聊天上下文窗口已经覆盖同对话信息;记忆读侧的价值主要在跨对话补充。
func buildReadScopedItemQuery(
req memorymodel.RetrieveRequest,
now time.Time,
statuses []string,
limit int,
) memorymodel.ItemQuery {
return memorymodel.ItemQuery{
UserID: req.UserID,
ConversationID: "",
AssistantID: req.AssistantID,
RunID: req.RunID,
Statuses: statuses,
MemoryTypes: normalizeRetrieveMemoryTypes(req.MemoryTypes),
IncludeGlobal: true,
OnlyUnexpired: true,
Limit: limit,
Now: now,
}
}
// buildReadScopedRAGRequest 构造读侧统一使用的 RAG 检索请求。
//
// 职责边界:
// 1. 只负责生成 memory 检索请求,不负责执行向量检索;
// 2. 不负责阈值外的重排、fallback 或去重;
// 3. conversation_id 字段同样只保留在文档 metadata 中,不再作为聊天读侧的硬过滤条件。
//
// 步骤化说明:
// 1. user_id 仍是唯一必须保留的硬过滤条件,确保召回范围限定在当前用户。
// 2. conversation_id 明确置空,避免旧对话记忆在进入相似度计算前就被 metadata filter 提前挡掉。
// 3. assistant_id / run_id 保持透传,方便后续若存在多助手场景时继续做更细粒度隔离。
func buildReadScopedRAGRequest(
req memorymodel.RetrieveRequest,
topK int,
threshold float64,
) ragservice.MemoryRetrieveRequest {
return ragservice.MemoryRetrieveRequest{
Query: req.Query,
TopK: topK,
Threshold: threshold,
Action: "search",
UserID: req.UserID,
ConversationID: "",
AssistantID: req.AssistantID,
RunID: req.RunID,
MemoryTypes: normalizeRetrieveMemoryTypes(req.MemoryTypes),
}
}
// shouldReturnSemanticRAGResult 判断当前是否可以直接采用 RAG 结果。
//
// 职责边界:
// 1. 只负责表达“RAG 是否足以短路后续 MySQL fallback”这一条业务规则
// 2. 不负责执行任何检索,也不负责日志记录;
// 3. 返回 false 不代表错误,只代表调用方应继续尝试数据库兜底。
//
// 步骤化说明:
// 1. RAG 报错时,一定不能短路,必须继续走 MySQL fallback。
// 2. RAG 0 命中时,同样不能短路;否则会把“成功执行但没有候选”误当成最终结果。
// 3. 只有“无报错且结果非空”时,才允许直接返回 RAG 结果。
func shouldReturnSemanticRAGResult(items []memorymodel.ItemDTO, err error) bool {
return err == nil && len(items) > 0
}

View File

@@ -0,0 +1,438 @@
package service
import (
"context"
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/LoveLosita/smartflow/backend/model"
memoryrepo "github.com/LoveLosita/smartflow/backend/services/memory/internal/repo"
memoryutils "github.com/LoveLosita/smartflow/backend/services/memory/internal/utils"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
memoryobserve "github.com/LoveLosita/smartflow/backend/services/memory/observe"
ragservice "github.com/LoveLosita/smartflow/backend/services/rag"
)
const (
defaultRetrieveLimit = 5
maxRetrieveLimit = 20
)
// ReadService 负责 memory 模块内部的读取、门控与轻量重排。
//
// 职责边界:
// 1. 负责把 memory_items 读出来并做用户设置过滤;
// 2. 负责最小可用的排序与截断,为后续 prompt 注入提供稳定入口;
// 3. 不直接依赖 agent不负责真正把记忆拼进 prompt。
type ReadService struct {
itemRepo *memoryrepo.ItemRepo
settingsRepo *memoryrepo.SettingsRepo
ragRuntime ragservice.Runtime
cfg memorymodel.Config
observer memoryobserve.Observer
metrics memoryobserve.MetricsRecorder
}
type retrieveTelemetry struct {
ReadMode string
QueryLen int
LegacyHitCount int
PinnedHitCount int
SemanticHitCount int
DedupDropCount int
FinalCount int
Degraded bool
RAGFallbackUsed bool
}
type semanticRetrieveTelemetry struct {
HitCount int
Degraded bool
RAGFallbackUsed bool
}
func NewReadService(
itemRepo *memoryrepo.ItemRepo,
settingsRepo *memoryrepo.SettingsRepo,
ragRuntime ragservice.Runtime,
cfg memorymodel.Config,
observer memoryobserve.Observer,
metrics memoryobserve.MetricsRecorder,
) *ReadService {
if observer == nil {
observer = memoryobserve.NewNopObserver()
}
if metrics == nil {
metrics = memoryobserve.NewNopMetrics()
}
return &ReadService{
itemRepo: itemRepo,
settingsRepo: settingsRepo,
ragRuntime: ragRuntime,
cfg: cfg,
observer: observer,
metrics: metrics,
}
}
// Retrieve 读取可供后续注入使用的候选记忆。
func (s *ReadService) Retrieve(ctx context.Context, req memorymodel.RetrieveRequest) ([]memorymodel.ItemDTO, error) {
if s == nil || s.itemRepo == nil || s.settingsRepo == nil {
return nil, nil
}
if req.UserID <= 0 {
return nil, nil
}
now := req.Now
if now.IsZero() {
now = time.Now()
}
telemetry := retrieveTelemetry{
ReadMode: s.cfg.EffectiveReadMode(),
QueryLen: len(strings.TrimSpace(req.Query)),
}
setting, err := s.settingsRepo.GetByUserID(ctx, req.UserID)
if err != nil {
s.recordRetrieve(ctx, req, telemetry, err)
return nil, err
}
effectiveSetting := memoryutils.EffectiveUserSetting(setting, req.UserID)
if !effectiveSetting.MemoryEnabled {
return nil, nil
}
limit := normalizeLimit(req.Limit, defaultRetrieveLimit, maxRetrieveLimit)
if s.cfg.EffectiveReadMode() == memorymodel.MemoryReadModeHybrid {
items, hybridTelemetry, hybridErr := s.HybridRetrieve(ctx, req, effectiveSetting, limit, now)
hybridTelemetry.ReadMode = memorymodel.MemoryReadModeHybrid
hybridTelemetry.QueryLen = telemetry.QueryLen
s.recordRetrieve(ctx, req, hybridTelemetry, hybridErr)
return items, hybridErr
}
if s.cfg.RAGEnabled && s.ragRuntime != nil && strings.TrimSpace(req.Query) != "" {
items, ragErr := s.retrieveByRAG(ctx, req, effectiveSetting, limit, now)
if ragErr == nil && len(items) > 0 {
telemetry.SemanticHitCount = len(items)
telemetry.FinalCount = len(items)
s.recordRetrieve(ctx, req, telemetry, nil)
return items, nil
}
telemetry.Degraded = true
telemetry.RAGFallbackUsed = true
}
items, legacyErr := s.retrieveByLegacy(ctx, req, limit, now, effectiveSetting)
telemetry.LegacyHitCount = len(items)
telemetry.FinalCount = len(items)
s.recordRetrieve(ctx, req, telemetry, legacyErr)
return items, legacyErr
}
func (s *ReadService) retrieveByLegacy(
ctx context.Context,
req memorymodel.RetrieveRequest,
limit int,
now time.Time,
effectiveSetting model.MemoryUserSetting,
) ([]memorymodel.ItemDTO, error) {
if !effectiveSetting.MemoryEnabled {
return nil, nil
}
query := buildReadScopedItemQuery(
req,
now,
[]string{model.MemoryItemStatusActive},
normalizeLimit(limit*3, limit*3, maxRetrieveLimit*3),
)
items, err := s.itemRepo.FindByQuery(ctx, query)
if err != nil {
return nil, err
}
items = memoryutils.FilterItemsBySetting(items, effectiveSetting)
if len(items) == 0 {
return nil, nil
}
sort.SliceStable(items, func(i, j int) bool {
left := scoreRetrievedItem(items[i], now)
right := scoreRetrievedItem(items[j], now)
if left == right {
return items[i].ID > items[j].ID
}
return left > right
})
if len(items) > limit {
items = items[:limit]
}
_ = s.itemRepo.TouchLastAccessAt(ctx, collectMemoryIDs(items), now)
return toItemDTOs(items), nil
}
func (s *ReadService) retrieveByRAG(
ctx context.Context,
req memorymodel.RetrieveRequest,
effectiveSetting model.MemoryUserSetting,
limit int,
now time.Time,
) ([]memorymodel.ItemDTO, error) {
if !effectiveSetting.MemoryEnabled {
return nil, nil
}
result, err := s.ragRuntime.RetrieveMemory(ctx, buildReadScopedRAGRequest(req, limit, s.cfg.Threshold))
if err != nil || result == nil || len(result.Items) == 0 {
return nil, err
}
items := make([]memorymodel.ItemDTO, 0, len(result.Items))
ids := make([]int64, 0, len(result.Items))
for _, hit := range result.Items {
dto, memoryID := buildMemoryDTOFromRetrieveHit(hit)
if !effectiveSetting.ImplicitMemoryEnabled && !dto.IsExplicit {
continue
}
if !effectiveSetting.SensitiveMemoryEnabled && dto.SensitivityLevel > 0 {
continue
}
if dto.ID <= 0 && memoryID > 0 {
dto.ID = memoryID
}
items = append(items, dto)
if dto.ID > 0 {
ids = append(ids, dto.ID)
}
}
if len(items) > limit {
items = items[:limit]
}
_ = s.itemRepo.TouchLastAccessAt(ctx, ids, now)
return items, nil
}
func normalizeRetrieveMemoryTypes(raw []string) []string {
normalized := normalizeMemoryTypes(raw)
if len(normalized) > 0 {
return normalized
}
return []string{
memorymodel.MemoryTypeConstraint,
memorymodel.MemoryTypePreference,
memorymodel.MemoryTypeFact,
}
}
func (s *ReadService) recordRetrieve(
ctx context.Context,
req memorymodel.RetrieveRequest,
telemetry retrieveTelemetry,
err error,
) {
if s == nil {
return
}
level := memoryobserve.LevelInfo
if err != nil {
level = memoryobserve.LevelWarn
}
s.observer.Observe(ctx, memoryobserve.Event{
Level: level,
Component: memoryobserve.ComponentRead,
Operation: memoryobserve.OperationRetrieve,
Fields: map[string]any{
"user_id": req.UserID,
"read_mode": telemetry.ReadMode,
"query_len": telemetry.QueryLen,
"legacy_hit_count": telemetry.LegacyHitCount,
"pinned_hit_count": telemetry.PinnedHitCount,
"semantic_hit_count": telemetry.SemanticHitCount,
"dedup_drop_count": telemetry.DedupDropCount,
"final_count": telemetry.FinalCount,
"degraded": telemetry.Degraded,
"rag_fallback_used": telemetry.RAGFallbackUsed,
"success": err == nil,
"error": err,
"error_code": memoryobserve.ClassifyError(err),
},
})
if telemetry.FinalCount > 0 {
s.metrics.AddCounter(memoryobserve.MetricRetrieveHitTotal, int64(telemetry.FinalCount), map[string]string{
"read_mode": strings.TrimSpace(telemetry.ReadMode),
})
}
if telemetry.DedupDropCount > 0 {
s.metrics.AddCounter(memoryobserve.MetricRetrieveDedupDropTotal, int64(telemetry.DedupDropCount), map[string]string{
"read_mode": strings.TrimSpace(telemetry.ReadMode),
})
}
if telemetry.RAGFallbackUsed {
s.metrics.AddCounter(memoryobserve.MetricRAGFallbackTotal, 1, map[string]string{
"read_mode": strings.TrimSpace(telemetry.ReadMode),
})
}
}
// scoreRetrievedItem 计算 legacy 读链路的确定性排序分数。
//
// 说明:
// 1. 这里只保留 importance / confidence / recency / explicit / type 这些稳定特征;
// 2. conversation_id 已不再参与读侧打分,因为同对话信息本就已经在上下文窗口内;
// 3. 若后续需要引入语义分或 reranker应在 DTO 层补齐对应字段后再统一并入。
func scoreRetrievedItem(item model.MemoryItem, now time.Time) float64 {
score := 0.35*clamp01(item.Importance) + 0.3*clamp01(item.Confidence) + 0.2*recencyScore(item, now)
if item.IsExplicit {
score += 0.1
}
switch item.MemoryType {
case memorymodel.MemoryTypeConstraint:
score += 0.12
case memorymodel.MemoryTypePreference:
score += 0.08
}
return score
}
func recencyScore(item model.MemoryItem, now time.Time) float64 {
base := item.UpdatedAt
if base == nil {
base = item.CreatedAt
}
if base == nil || now.Before(*base) {
return 0.5
}
age := now.Sub(*base)
switch {
case age <= 24*time.Hour:
return 1
case age <= 7*24*time.Hour:
return 0.85
case age <= 30*24*time.Hour:
return 0.65
case age <= 90*24*time.Hour:
return 0.45
default:
return 0.25
}
}
func clamp01(v float64) float64 {
if v < 0 {
return 0
}
if v > 1 {
return 1
}
return v
}
func collectMemoryIDs(items []model.MemoryItem) []int64 {
if len(items) == 0 {
return nil
}
ids := make([]int64, 0, len(items))
for _, item := range items {
if item.ID <= 0 {
continue
}
ids = append(ids, item.ID)
}
return ids
}
func buildMemoryDTOFromRetrieveHit(hit ragservice.RetrieveHit) (memorymodel.ItemDTO, int64) {
memoryID := parseMemoryIDFromDocumentID(hit.DocumentID)
metadata := hit.Metadata
content := strings.TrimSpace(hit.Text)
memoryType := readString(metadata["memory_type"])
dto := memorymodel.ItemDTO{
ID: memoryID,
UserID: int(readFloatLike(metadata["user_id"])),
ConversationID: readString(metadata["conversation_id"]),
AssistantID: readString(metadata["assistant_id"]),
RunID: readString(metadata["run_id"]),
MemoryType: memoryType,
Title: readString(metadata["title"]),
Content: content,
ContentHash: fallbackContentHash(memoryType, content, readString(metadata["content_hash"])),
Confidence: readFloatLike(metadata["confidence"]),
Importance: readFloatLike(metadata["importance"]),
SensitivityLevel: int(readFloatLike(metadata["sensitivity_level"])),
IsExplicit: readBoolLike(metadata["is_explicit"]),
Status: readString(metadata["status"]),
TTLAt: readTimeLike(metadata["ttl_at"]),
}
return dto, memoryID
}
func parseMemoryIDFromDocumentID(documentID string) int64 {
documentID = strings.TrimSpace(documentID)
if !strings.HasPrefix(documentID, "memory:") {
return 0
}
raw := strings.TrimPrefix(documentID, "memory:")
if strings.HasPrefix(raw, "uid:") {
return 0
}
parsed, err := strconv.ParseInt(raw, 10, 64)
if err != nil {
return 0
}
return parsed
}
func readString(v any) string {
if v == nil {
return ""
}
return strings.TrimSpace(fmt.Sprintf("%v", v))
}
func readFloatLike(v any) float64 {
switch value := v.(type) {
case float64:
return value
case float32:
return float64(value)
case int:
return float64(value)
case int64:
return float64(value)
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
if err == nil {
return parsed
}
}
return 0
}
func readBoolLike(v any) bool {
switch value := v.(type) {
case bool:
return value
case string:
return strings.EqualFold(strings.TrimSpace(value), "true")
default:
return false
}
}
func readTimeLike(v any) *time.Time {
text := readString(v)
if text == "" {
return nil
}
parsed, err := time.Parse(time.RFC3339, text)
if err != nil {
return nil
}
return &parsed
}

View File

@@ -0,0 +1,341 @@
package service
import (
"context"
"strings"
"time"
"github.com/LoveLosita/smartflow/backend/model"
memoryutils "github.com/LoveLosita/smartflow/backend/services/memory/internal/utils"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
// HybridRetrieve 统一承接读取侧 RAG-first 召回链路。
//
// 步骤化说明:
// 1. 优先走 RAG 语义搜索,按 query 相关性召回候选记忆;
// 2. RAG 报错或 0 命中时回退 MySQL保证链路韧性
// 3. 召回结果做三级去重、排序与类型预算裁剪(总量不超过调用方 limit
// 4. 旧 legacy 链路完全保留,方便通过配置快速回滚。
func (s *ReadService) HybridRetrieve(
ctx context.Context,
req memorymodel.RetrieveRequest,
effectiveSetting model.MemoryUserSetting,
limit int,
now time.Time,
) ([]memorymodel.ItemDTO, retrieveTelemetry, error) {
telemetry := retrieveTelemetry{}
if s == nil || s.itemRepo == nil {
return nil, telemetry, nil
}
if !effectiveSetting.MemoryEnabled {
return nil, telemetry, nil
}
// RAG-first只走语义召回不再全量拉 MySQL pinned。
items, semanticTelemetry, err := s.retrieveSemanticCandidates(ctx, req, effectiveSetting, limit, now)
if err != nil {
return nil, telemetry, err
}
telemetry.SemanticHitCount = semanticTelemetry.HitCount
telemetry.Degraded = semanticTelemetry.Degraded
telemetry.RAGFallbackUsed = semanticTelemetry.RAGFallbackUsed
if len(items) == 0 {
return nil, telemetry, nil
}
beforeDedupCount := len(items)
items = dedupByID(items)
items = dedupByHash(items)
items = dedupByText(items)
telemetry.DedupDropCount = beforeDedupCount - len(items)
items = RankItems(items, now)
items = applyTypeBudget(items, s.cfg, limit)
if len(items) == 0 {
return nil, telemetry, nil
}
telemetry.FinalCount = len(items)
_ = s.itemRepo.TouchLastAccessAt(ctx, collectItemDTOIDs(items), now)
return items, telemetry, nil
}
func (s *ReadService) retrievePinnedCandidates(
ctx context.Context,
req memorymodel.RetrieveRequest,
effectiveSetting model.MemoryUserSetting,
now time.Time,
) ([]memorymodel.ItemDTO, error) {
query := buildReadScopedItemQuery(req, now, nil, 0)
items, err := s.itemRepo.FindPinnedByUser(ctx, query, s.cfg.EffectiveReadPreferenceLimit())
if err != nil {
return nil, err
}
items = memoryutils.FilterItemsBySetting(items, effectiveSetting)
return toItemDTOs(items), nil
}
func (s *ReadService) retrieveSemanticCandidates(
ctx context.Context,
req memorymodel.RetrieveRequest,
effectiveSetting model.MemoryUserSetting,
limit int,
now time.Time,
) ([]memorymodel.ItemDTO, semanticRetrieveTelemetry, error) {
telemetry := semanticRetrieveTelemetry{}
queryText := strings.TrimSpace(req.Query)
if queryText == "" {
return nil, telemetry, nil
}
candidateLimit := hybridSemanticTopK(s.cfg, limit)
if s.cfg.RAGEnabled && s.ragRuntime != nil {
items, err := s.retrieveSemanticCandidatesByRAG(ctx, req, effectiveSetting, candidateLimit, now)
if shouldReturnSemanticRAGResult(items, err) {
telemetry.HitCount = len(items)
return items, telemetry, nil
}
telemetry.Degraded = true
telemetry.RAGFallbackUsed = true
}
items, err := s.retrieveSemanticCandidatesByMySQL(ctx, req, effectiveSetting, candidateLimit, now)
telemetry.HitCount = len(items)
return items, telemetry, err
}
func (s *ReadService) retrieveSemanticCandidatesByRAG(
ctx context.Context,
req memorymodel.RetrieveRequest,
effectiveSetting model.MemoryUserSetting,
candidateLimit int,
now time.Time,
) ([]memorymodel.ItemDTO, error) {
result, err := s.ragRuntime.RetrieveMemory(ctx, buildReadScopedRAGRequest(req, candidateLimit, s.cfg.Threshold))
if err != nil {
return nil, err
}
if result == nil || len(result.Items) == 0 {
return nil, nil
}
items := make([]memorymodel.ItemDTO, 0, len(result.Items))
for _, hit := range result.Items {
dto, memoryID := buildMemoryDTOFromRetrieveHit(hit)
if !effectiveSetting.ImplicitMemoryEnabled && !dto.IsExplicit {
continue
}
if !effectiveSetting.SensitiveMemoryEnabled && dto.SensitivityLevel > 0 {
continue
}
if dto.ID <= 0 && memoryID > 0 {
dto.ID = memoryID
}
items = append(items, dto)
}
return items, nil
}
func (s *ReadService) retrieveSemanticCandidatesByMySQL(
ctx context.Context,
req memorymodel.RetrieveRequest,
effectiveSetting model.MemoryUserSetting,
candidateLimit int,
now time.Time,
) ([]memorymodel.ItemDTO, error) {
query := buildReadScopedItemQuery(
req,
now,
[]string{model.MemoryItemStatusActive},
normalizeLimit(candidateLimit, candidateLimit, maxRetrieveLimit),
)
items, err := s.itemRepo.FindByQuery(ctx, query)
if err != nil {
return nil, err
}
items = memoryutils.FilterItemsBySetting(items, effectiveSetting)
return toItemDTOs(items), nil
}
// dedupByID 按 memory_id 去重,后出现的结果覆盖先出现的结果。
func dedupByID(items []memorymodel.ItemDTO) []memorymodel.ItemDTO {
if len(items) == 0 {
return nil
}
seen := make(map[int64]struct{}, len(items))
result := make([]memorymodel.ItemDTO, 0, len(items))
for i := len(items) - 1; i >= 0; i-- {
item := items[i]
if item.ID <= 0 {
result = append(result, item)
continue
}
if _, exists := seen[item.ID]; exists {
continue
}
seen[item.ID] = struct{}{}
result = append(result, item)
}
reverseItemDTOs(result)
return result
}
// dedupByHash 按 content_hash 去重;缺失 hash 时跳过,保留 importance 更高的条目。
func dedupByHash(items []memorymodel.ItemDTO) []memorymodel.ItemDTO {
return dedupByKey(items, func(item memorymodel.ItemDTO) string {
return fallbackContentHash(item.MemoryType, item.Content, item.ContentHash)
})
}
// dedupByText 按“类型标签 + 文本”兜底去重,用于覆盖历史数据未带 hash 的场景。
func dedupByText(items []memorymodel.ItemDTO) []memorymodel.ItemDTO {
return dedupByKey(items, func(item memorymodel.ItemDTO) string {
text := strings.TrimSpace(item.Content)
if text == "" {
text = strings.TrimSpace(item.Title)
}
if text == "" {
return ""
}
return renderMemoryTypeLabelForDedup(item.MemoryType) + "::" + normalizeContentForHash(text)
})
}
func dedupByKey(items []memorymodel.ItemDTO, keyBuilder func(item memorymodel.ItemDTO) string) []memorymodel.ItemDTO {
if len(items) == 0 {
return nil
}
selectedIndex := make(map[string]int, len(items))
for index, item := range items {
key := strings.TrimSpace(keyBuilder(item))
if key == "" {
continue
}
if previous, exists := selectedIndex[key]; exists {
if preferCurrentItem(items[previous], item) {
selectedIndex[key] = index
}
continue
}
selectedIndex[key] = index
}
result := make([]memorymodel.ItemDTO, 0, len(items))
for index, item := range items {
key := strings.TrimSpace(keyBuilder(item))
if key == "" {
result = append(result, item)
continue
}
if selectedIndex[key] == index {
result = append(result, item)
}
}
return result
}
func preferCurrentItem(previous memorymodel.ItemDTO, current memorymodel.ItemDTO) bool {
if current.Importance != previous.Importance {
return current.Importance > previous.Importance
}
if current.Confidence != previous.Confidence {
return current.Confidence > previous.Confidence
}
return true
}
// applyTypeBudget 在排序结果上应用四类记忆预算,并以 callerLimit 作为总量硬上限。
//
// 说明:
// 1. 每种类型先保底自己的预算上限,避免 fact 抢掉 constraint 的位置;
// 2. 裁剪时保持当前排序顺序,不在这里重新打分;
// 3. 最终总量不超过 min(callerLimit, cfg.TotalReadBudget())。
func applyTypeBudget(items []memorymodel.ItemDTO, cfg memorymodel.Config, callerLimit int) []memorymodel.ItemDTO {
if len(items) == 0 {
return nil
}
hardCap := cfg.TotalReadBudget()
if callerLimit > 0 && callerLimit < hardCap {
hardCap = callerLimit
}
budgetByType := map[string]int{
memorymodel.MemoryTypeConstraint: cfg.EffectiveReadConstraintLimit(),
memorymodel.MemoryTypePreference: cfg.EffectiveReadPreferenceLimit(),
memorymodel.MemoryTypeFact: cfg.EffectiveReadFactLimit(),
}
usedByType := make(map[string]int, len(budgetByType))
result := make([]memorymodel.ItemDTO, 0, minInt(len(items), hardCap))
for _, item := range items {
if len(result) >= hardCap {
break
}
memoryType := resolveBudgetMemoryType(item.MemoryType)
if usedByType[memoryType] >= budgetByType[memoryType] {
continue
}
usedByType[memoryType]++
result = append(result, item)
}
return result
}
// hybridSemanticTopK 计算语义召回的候选集大小。
// 使用 callerLimit 的 2 倍作为 TopK保证去重后仍有足够结果填充预算。
func hybridSemanticTopK(cfg memorymodel.Config, limit int) int {
return limit * 2
}
func resolveBudgetMemoryType(memoryType string) string {
normalized := memorymodel.NormalizeMemoryType(memoryType)
if normalized == "" {
return memorymodel.MemoryTypeFact
}
return normalized
}
func renderMemoryTypeLabelForDedup(memoryType string) string {
switch memorymodel.NormalizeMemoryType(memoryType) {
case memorymodel.MemoryTypePreference:
return "偏好"
case memorymodel.MemoryTypeConstraint:
return "约束"
case memorymodel.MemoryTypeFact:
return "事实"
default:
return "记忆"
}
}
func collectItemDTOIDs(items []memorymodel.ItemDTO) []int64 {
if len(items) == 0 {
return nil
}
ids := make([]int64, 0, len(items))
for _, item := range items {
if item.ID <= 0 {
continue
}
ids = append(ids, item.ID)
}
return ids
}
func reverseItemDTOs(items []memorymodel.ItemDTO) {
for left, right := 0, len(items)-1; left < right; left, right = left+1, right-1 {
items[left], items[right] = items[right], items[left]
}
}
func minInt(left, right int) int {
if left < right {
return left
}
return right
}

View File

@@ -0,0 +1,76 @@
package service
import (
"sort"
"time"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
// RankItems 对读取结果做统一重排。
//
// 步骤化说明:
// 1. 先基于 importance / confidence / recency 构造基础分,保持和旧链路相近的排序直觉;
// 2. 再叠加“显式记忆 / 类型优先级”奖励,让 constraint 与 preference 更稳定地排在前面;
// 3. 同分按 ID 降序,保证排序在日志与测试里具备稳定性。
func RankItems(items []memorymodel.ItemDTO, now time.Time) []memorymodel.ItemDTO {
if len(items) == 0 {
return nil
}
ranked := make([]memorymodel.ItemDTO, len(items))
copy(ranked, items)
sort.SliceStable(ranked, func(i, j int) bool {
left := scoreRankedItem(ranked[i], now)
right := scoreRankedItem(ranked[j], now)
if left == right {
return ranked[i].ID > ranked[j].ID
}
return left > right
})
return ranked
}
// scoreRankedItem 计算 hybrid 读链路的统一重排分数。
//
// 说明:
// 1. 这里仍然只依赖条目自身属性,不引入 conversation_id 加分;
// 2. 原因是同对话内容本就已经存在于上下文窗口,记忆读侧应专注跨对话补充;
// 3. 类型加权仍然保留,用于确保 constraint / preference 的业务优先级稳定生效。
func scoreRankedItem(item memorymodel.ItemDTO, now time.Time) float64 {
score := 0.35*clamp01(item.Importance) + 0.3*clamp01(item.Confidence) + 0.2*recencyScoreDTO(item, now)
if item.IsExplicit {
score += 0.1
}
switch memorymodel.NormalizeMemoryType(item.MemoryType) {
case memorymodel.MemoryTypeConstraint:
score += 0.15
case memorymodel.MemoryTypePreference:
score += 0.10
}
return score
}
func recencyScoreDTO(item memorymodel.ItemDTO, now time.Time) float64 {
base := item.UpdatedAt
if base == nil {
base = item.CreatedAt
}
if base == nil || now.Before(*base) {
return 0.5
}
age := now.Sub(*base)
switch {
case age <= 24*time.Hour:
return 1
case age <= 7*24*time.Hour:
return 0.85
case age <= 30*24*time.Hour:
return 0.65
case age <= 90*24*time.Hour:
return 0.45
default:
return 0.25
}
}

View File

@@ -0,0 +1,115 @@
package utils
import (
"fmt"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
// AggregateComparisons 把一轮 LLM 比对结果汇总为最终动作。
//
// 职责边界:
// 1. 纯确定性逻辑,不调 LLM不调外部服务
// 2. 按优先级从高到低判定duplicate > update > conflict > unrelated
// 3. 多条 update 时选 Score 最高的候选执行 UPDATE。
//
// 汇总规则:
// 1. 出现 duplicate → 最终动作 NONE新 fact 完全重复,不需要写入);
// 2. 出现 update → 最终动作 UPDATE更新 Score 最高的那条旧记忆);
// 3. 出现 conflict → 最终动作 DELETE + 后续按 ADD 处理(旧记忆过时,先删旧的再写新的);
// 4. 全部 unrelated → 最终动作 ADD没有相关旧记忆直接新增
func AggregateComparisons(
fact memorymodel.NormalizedFact,
comparisons []memorymodel.ComparisonResult,
candidates []memorymodel.CandidateSnapshot,
) *memorymodel.FinalDecision {
// 1. 无候选时直接 ADD无需走任何判断。
if len(comparisons) == 0 {
return &memorymodel.FinalDecision{
Action: memorymodel.DecisionActionAdd,
Reason: "无相关旧记忆,直接新增",
}
}
// 2. 建立 memoryID → CandidateSnapshot 映射,用于查找 Score。
snapshotMap := make(map[int64]memorymodel.CandidateSnapshot, len(candidates))
for _, c := range candidates {
snapshotMap[c.MemoryID] = c
}
hasDuplicate := false
var bestUpdate *memorymodel.ComparisonResult
bestUpdateScore := -1.0
var conflictResult *memorymodel.ComparisonResult
for i := range comparisons {
comp := &comparisons[i]
switch comp.Relation {
case memorymodel.RelationDuplicate:
// 3. 出现一条 duplicate 即可确定最终动作为 NONE。
hasDuplicate = true
case memorymodel.RelationUpdate:
// 4. 多条 update 时,选 Score 最高的那条执行 UPDATE。
snapshot, ok := snapshotMap[comp.MemoryID]
score := 0.0
if ok {
score = snapshot.Score
}
if score > bestUpdateScore {
bestUpdateScore = score
bestUpdate = comp
}
case memorymodel.RelationConflict:
// 5. 记录第一条 conflict用于后续 DELETE + ADD 处理。
if conflictResult == nil {
conflictResult = comp
}
}
}
// 6. 按优先级判定最终动作。
if hasDuplicate {
return &memorymodel.FinalDecision{
Action: memorymodel.DecisionActionNone,
Reason: "存在完全重复的旧记忆,跳过写入",
}
}
if bestUpdate != nil {
// 7. UPDATE 动作:使用 LLM 提供的合并后内容。
title := bestUpdate.UpdatedTitle
if title == "" {
title = fact.Title
}
content := bestUpdate.UpdatedContent
reason := bestUpdate.Reason
if reason == "" {
reason = "新事实是对旧记忆的修正或补充"
}
return &memorymodel.FinalDecision{
Action: memorymodel.DecisionActionUpdate,
TargetID: bestUpdate.MemoryID,
Title: title,
Content: content,
Reason: fmt.Sprintf("更新旧记忆(id=%d): %s", bestUpdate.MemoryID, reason),
}
}
if conflictResult != nil {
// 8. conflict → 先 DELETE 旧记忆,后续由上层按 ADD 写入新 fact。
return &memorymodel.FinalDecision{
Action: memorymodel.DecisionActionDelete,
TargetID: conflictResult.MemoryID,
Reason: fmt.Sprintf("旧记忆(id=%d)与新事实冲突,删除后新增: %s", conflictResult.MemoryID, conflictResult.Reason),
}
}
// 9. 全部 unrelated → 直接 ADD。
return &memorymodel.FinalDecision{
Action: memorymodel.DecisionActionAdd,
Reason: "无相关旧记忆,直接新增",
}
}

View File

@@ -0,0 +1,77 @@
package utils
import (
"encoding/json"
"strings"
"github.com/LoveLosita/smartflow/backend/model"
)
const (
// AuditOperationCreate 表示系统新建一条记忆。
AuditOperationCreate = "create"
// AuditOperationUpdate 表示决策层更新已有记忆的内容。
AuditOperationUpdate = "update"
// AuditOperationArchive 表示治理层把重复记忆归档。
AuditOperationArchive = "archive"
// AuditOperationDelete 表示对已有记忆做软删除。
AuditOperationDelete = "delete"
// AuditOperationRestore 表示把已删除/归档记忆恢复为 active。
AuditOperationRestore = "restore"
)
// BuildItemAuditLog 构造记忆变更审计日志。
//
// 职责边界:
// 1. 负责把 before/after 快照统一序列化为审计日志结构;
// 2. 不负责决定“是否应该写审计”,该决策由上层 service/worker 控制;
// 3. 不负责落库,调用方仍需显式调用 AuditRepo。
func BuildItemAuditLog(
memoryID int64,
userID int,
operation string,
operatorType string,
reason string,
before *model.MemoryItem,
after *model.MemoryItem,
) model.MemoryAuditLog {
return model.MemoryAuditLog{
MemoryID: memoryID,
UserID: userID,
Operation: strings.TrimSpace(operation),
OperatorType: NormalizeOperatorType(operatorType),
Reason: strings.TrimSpace(reason),
BeforeJSON: marshalMemoryItemSnapshot(before),
AfterJSON: marshalMemoryItemSnapshot(after),
}
}
// NormalizeOperatorType 统一规整审计操作者类型。
//
// 规则说明:
// 1. 目前只接受 user/system 两类固定值;
// 2. 空值或未知值统一回退为 user避免把脏值直接写进审计表
// 3. 若后续扩展 admin/tool 等类型,再在这里集中放开即可。
func NormalizeOperatorType(raw string) string {
switch strings.ToLower(strings.TrimSpace(raw)) {
case "system":
return "system"
default:
return "user"
}
}
func marshalMemoryItemSnapshot(item *model.MemoryItem) *string {
if item == nil {
return nil
}
raw, err := json.Marshal(item)
if err != nil {
empty := "{}"
return &empty
}
value := string(raw)
return &value
}

View File

@@ -0,0 +1,49 @@
package utils
import (
"fmt"
"strings"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
// 合法关系类型集合,用于校验 LLM 输出的 relation 字段。
var validRelations = map[string]struct{}{
memorymodel.RelationDuplicate: {},
memorymodel.RelationUpdate: {},
memorymodel.RelationConflict: {},
memorymodel.RelationUnrelated: {},
}
// ValidateComparisonResult 校验单次比对结果的基本合法性。
//
// 职责边界:
// 1. 只校验 LLM 输出的结构合法性,不校验业务语义;
// 2. relation 必须是四种合法值之一update 时必须有 UpdatedContent
// 3. 校验失败直接返回 error调用方决定丢弃或重试。
func ValidateComparisonResult(result *memorymodel.ComparisonResult) error {
if result == nil {
return fmt.Errorf("比对结果不能为空")
}
// 1. MemoryID 必须大于 0确保能定位到旧记忆。
if result.MemoryID <= 0 {
return fmt.Errorf("比对结果 memory_id 无效: %d", result.MemoryID)
}
// 2. relation 必须是四种合法值之一,防止 LLM 输出非法值。
relation := strings.TrimSpace(strings.ToLower(result.Relation))
if _, ok := validRelations[relation]; !ok {
return fmt.Errorf("比对结果 relation 非法: %s", result.Relation)
}
// 3. relation=update 时UpdatedContent 不能为空。
// 原因update 需要合并后的完整内容,不能只写差异部分。
if relation == memorymodel.RelationUpdate {
if strings.TrimSpace(result.UpdatedContent) == "" {
return fmt.Errorf("relation=update 时 updated_content 不能为空")
}
}
return nil
}

View File

@@ -0,0 +1,104 @@
package utils
import (
"encoding/json"
"errors"
"regexp"
"strings"
)
var fencedJSONPattern = regexp.MustCompile("(?s)```(?:json)?\\s*([\\[{].*[\\]}])\\s*```")
// ExtractJSON 从模型输出中提取 JSON 文本(兼容代码块包裹)。
//
// 步骤:
// 1. 先判断整段文本是否本身就是合法 JSON
// 2. 再尝试匹配 ```json ... ``` 代码块;
// 3. 最后做一次“首个 JSON 对象/数组”扫描提取。
func ExtractJSON(raw string) (string, error) {
trimmed := strings.TrimSpace(raw)
if trimmed == "" {
return "", errors.New("empty model output")
}
// 1. 直接 JSON 命中时,避免做额外启发式扫描。
if json.Valid([]byte(trimmed)) {
return trimmed, nil
}
// 2. 兼容 markdown 代码块包裹 JSON。
matches := fencedJSONPattern.FindStringSubmatch(trimmed)
if len(matches) > 1 {
candidate := strings.TrimSpace(matches[1])
if json.Valid([]byte(candidate)) {
return candidate, nil
}
}
// 3. 兜底扫描首个完整 JSON 片段,尽量提升容错能力。
if candidate, ok := findFirstJSONSegment(trimmed); ok {
return candidate, nil
}
return "", errors.New("json not found in model output")
}
func findFirstJSONSegment(raw string) (string, bool) {
start := -1
var open, close rune
for i, ch := range raw {
if ch == '{' {
start = i
open = '{'
close = '}'
break
}
if ch == '[' {
start = i
open = '['
close = ']'
break
}
}
if start < 0 {
return "", false
}
depth := 0
inString := false
escaped := false
for i, ch := range raw[start:] {
if inString {
if escaped {
escaped = false
continue
}
if ch == '\\' {
escaped = true
continue
}
if ch == '"' {
inString = false
}
continue
}
if ch == '"' {
inString = true
continue
}
if ch == open {
depth++
continue
}
if ch == close {
depth--
if depth == 0 {
candidate := strings.TrimSpace(raw[start : start+i+1])
if json.Valid([]byte(candidate)) {
return candidate, true
}
return "", false
}
}
}
return "", false
}

View File

@@ -0,0 +1,133 @@
package utils
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"strings"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
const (
maxTitleLength = 64
maxContentLength = 1000
)
// NormalizeFacts 对候选事实做标准化与过滤。
//
// 步骤:
// 1. 标准化 memory_type 与文本字段,丢弃空值和非法类型;
// 2. 对超长内容截断,避免脏数据污染后续链路;
// 3. 基于“类型+标准化内容”做去重,避免同一轮重复写入。
func NormalizeFacts(candidates []memorymodel.FactCandidate) []memorymodel.NormalizedFact {
if len(candidates) == 0 {
return nil
}
result := make([]memorymodel.NormalizedFact, 0, len(candidates))
seen := make(map[string]struct{}, len(candidates))
for _, candidate := range candidates {
memoryType := memorymodel.NormalizeMemoryType(candidate.MemoryType)
if memoryType == "" {
continue
}
content := normalizeWhitespace(candidate.Content)
if content == "" {
continue
}
content = truncateByRune(content, maxContentLength)
title := normalizeWhitespace(candidate.Title)
if title == "" {
title = truncateByRune(content, maxTitleLength)
}
title = truncateByRune(title, maxTitleLength)
confidence := clamp01(candidate.Confidence)
if confidence == 0 {
confidence = 0.6
}
importance := clamp01(candidate.Importance)
if importance == 0 {
importance = defaultImportanceByType(memoryType)
}
sensitivityLevel := clampInt(candidate.SensitivityLevel, 0, 2)
normalizedContent := strings.ToLower(content)
contentHash := HashContent(memoryType, normalizedContent)
dedupKey := fmt.Sprintf("%s:%s", memoryType, contentHash)
if _, exists := seen[dedupKey]; exists {
continue
}
seen[dedupKey] = struct{}{}
result = append(result, memorymodel.NormalizedFact{
MemoryType: memoryType,
Title: title,
Content: content,
NormalizedContent: normalizedContent,
ContentHash: contentHash,
Confidence: confidence,
Importance: importance,
SensitivityLevel: sensitivityLevel,
IsExplicit: candidate.IsExplicit,
})
}
return result
}
func normalizeWhitespace(raw string) string {
return strings.Join(strings.Fields(strings.TrimSpace(raw)), " ")
}
func truncateByRune(raw string, max int) string {
if max <= 0 {
return ""
}
runes := []rune(raw)
if len(runes) <= max {
return raw
}
return string(runes[:max])
}
func clamp01(v float64) float64 {
if v < 0 {
return 0
}
if v > 1 {
return 1
}
return v
}
func clampInt(v, minValue, maxValue int) int {
if v < minValue {
return minValue
}
if v > maxValue {
return maxValue
}
return v
}
func defaultImportanceByType(memoryType string) float64 {
switch memoryType {
case memorymodel.MemoryTypePreference:
return 0.85
case memorymodel.MemoryTypeConstraint:
return 0.95
default:
return 0.6
}
}
// HashContent 计算记忆内容的去重哈希。
// 算法sha256(memoryType + "::" + normalizedContent)
// 说明:导出此函数是为了让决策层 apply_actions 也能复用同一算法,避免哈希不一致导致去重失效。
func HashContent(memoryType, normalizedContent string) string {
sum := sha256.Sum256([]byte(memoryType + "::" + normalizedContent))
return hex.EncodeToString(sum[:])
}

View File

@@ -0,0 +1,81 @@
package utils
import (
"github.com/LoveLosita/smartflow/backend/model"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
// EffectiveUserSetting 返回用户记忆设置的生效值。
//
// 规则说明:
// 1. 用户未显式配置时,走系统默认值;
// 2. 默认允许普通记忆和隐式记忆,但默认关闭敏感记忆;
// 3. 返回值始终是完整对象,方便调用方直接使用,不再分支判空。
func EffectiveUserSetting(setting *model.MemoryUserSetting, userID int) model.MemoryUserSetting {
if setting == nil {
return model.MemoryUserSetting{
UserID: userID,
MemoryEnabled: true,
ImplicitMemoryEnabled: true,
SensitiveMemoryEnabled: false,
}
}
return *setting
}
// FilterFactsBySetting 按用户记忆开关过滤候选事实。
func FilterFactsBySetting(facts []memorymodel.NormalizedFact, setting model.MemoryUserSetting) []memorymodel.NormalizedFact {
if !setting.MemoryEnabled || len(facts) == 0 {
return nil
}
result := make([]memorymodel.NormalizedFact, 0, len(facts))
for _, fact := range facts {
if !setting.ImplicitMemoryEnabled && !fact.IsExplicit {
continue
}
if !setting.SensitiveMemoryEnabled && fact.SensitivityLevel > 0 {
continue
}
result = append(result, fact)
}
return result
}
// FilterItemsBySetting 按用户记忆开关过滤已入库记忆。
func FilterItemsBySetting(items []model.MemoryItem, setting model.MemoryUserSetting) []model.MemoryItem {
if !setting.MemoryEnabled || len(items) == 0 {
return nil
}
result := make([]model.MemoryItem, 0, len(items))
for _, item := range items {
if !setting.ImplicitMemoryEnabled && !item.IsExplicit {
continue
}
if !setting.SensitiveMemoryEnabled && item.SensitivityLevel > 0 {
continue
}
result = append(result, item)
}
return result
}
// FilterFactsByConfidence 按置信度阈值过滤候选事实。
//
// 说明:
// 1. minConfidence <= 0 时不做过滤,保持向后兼容;
// 2. 过滤在 FilterFactsBySetting 之后执行,是写入链路的第二道程序化门槛;
// 3. 阈值由 memory.write.minConfidence 配置控制,默认 0.5。
func FilterFactsByConfidence(facts []memorymodel.NormalizedFact, minConfidence float64) []memorymodel.NormalizedFact {
if minConfidence <= 0 || len(facts) == 0 {
return facts
}
result := make([]memorymodel.NormalizedFact, 0, len(facts))
for _, fact := range facts {
if fact.Confidence >= minConfidence {
result = append(result, fact)
}
}
return result
}

View File

@@ -0,0 +1,213 @@
package vectorsync
import (
"context"
"fmt"
"log"
"strings"
"github.com/LoveLosita/smartflow/backend/model"
memoryrepo "github.com/LoveLosita/smartflow/backend/services/memory/internal/repo"
memoryobserve "github.com/LoveLosita/smartflow/backend/services/memory/observe"
ragservice "github.com/LoveLosita/smartflow/backend/services/rag"
)
// Syncer 负责 memory_items 与向量库之间的最小桥接。
//
// 职责边界:
// 1. 只负责“把已经落库的记忆同步到 RAG / 从 RAG 删除”;
// 2. 不负责决定哪些记忆该写、该删、该恢复,这些决策仍由上游 service/worker/cleanup 控制;
// 3. 同步失败时只回写 vector_status 并打观测,不反向回滚业务事务,避免把在线链路拖成强依赖。
type Syncer struct {
ragRuntime ragservice.Runtime
itemRepo *memoryrepo.ItemRepo
observer memoryobserve.Observer
metrics memoryobserve.MetricsRecorder
logger *log.Logger
}
func NewSyncer(
ragRuntime ragservice.Runtime,
itemRepo *memoryrepo.ItemRepo,
observer memoryobserve.Observer,
metrics memoryobserve.MetricsRecorder,
) *Syncer {
if observer == nil {
observer = memoryobserve.NewNopObserver()
}
if metrics == nil {
metrics = memoryobserve.NewNopMetrics()
}
return &Syncer{
ragRuntime: ragRuntime,
itemRepo: itemRepo,
observer: observer,
metrics: metrics,
logger: log.Default(),
}
}
// Upsert 把新增/修改/恢复后的记忆同步到向量库。
func (s *Syncer) Upsert(ctx context.Context, traceID string, items []model.MemoryItem) {
if s == nil || s.ragRuntime == nil || s.itemRepo == nil || len(items) == 0 {
return
}
requestItems := make([]ragservice.MemoryIngestItem, 0, len(items))
for _, item := range items {
requestItems = append(requestItems, ragservice.MemoryIngestItem{
MemoryID: item.ID,
UserID: item.UserID,
ConversationID: strValue(item.ConversationID),
AssistantID: strValue(item.AssistantID),
RunID: strValue(item.RunID),
MemoryType: item.MemoryType,
Title: item.Title,
Content: item.Content,
Confidence: item.Confidence,
Importance: item.Importance,
SensitivityLevel: item.SensitivityLevel,
IsExplicit: item.IsExplicit,
Status: item.Status,
TTLAt: item.TTLAt,
CreatedAt: item.CreatedAt,
})
}
result, err := s.ragRuntime.IngestMemory(memoryobserve.WithFields(ctx, map[string]any{
"trace_id": traceID,
}), ragservice.MemoryIngestRequest{
TraceID: traceID,
Action: "add",
Items: requestItems,
})
if err != nil {
s.observer.Observe(ctx, memoryobserve.Event{
Level: memoryobserve.LevelWarn,
Component: memoryobserve.ComponentWrite,
Operation: "vector_upsert",
Fields: map[string]any{
"trace_id": traceID,
"item_count": len(items),
"success": false,
"error": err,
"error_code": memoryobserve.ClassifyError(err),
},
})
for _, item := range items {
_ = s.itemRepo.UpdateVectorStateByID(ctx, item.ID, "failed", nil)
}
return
}
vectorIDMap := make(map[int64]string, len(result.DocumentIDs))
for _, documentID := range result.DocumentIDs {
memoryID := parseMemoryID(documentID)
if memoryID <= 0 {
continue
}
vectorIDMap[memoryID] = documentID
}
for _, item := range items {
vectorID := strPtrOrNil(vectorIDMap[item.ID])
_ = s.itemRepo.UpdateVectorStateByID(ctx, item.ID, "synced", vectorID)
}
s.observer.Observe(ctx, memoryobserve.Event{
Level: memoryobserve.LevelInfo,
Component: memoryobserve.ComponentWrite,
Operation: "vector_upsert",
Fields: map[string]any{
"trace_id": traceID,
"item_count": len(items),
"document_count": len(result.DocumentIDs),
"success": true,
},
})
}
// Delete 把一批记忆对应的向量从向量库中删除。
func (s *Syncer) Delete(ctx context.Context, traceID string, memoryIDs []int64) {
if s == nil || len(memoryIDs) == 0 {
return
}
if s.ragRuntime == nil || s.itemRepo == nil {
return
}
documentIDs := make([]string, 0, len(memoryIDs))
for _, id := range memoryIDs {
documentIDs = append(documentIDs, fmt.Sprintf("memory:%d", id))
}
err := s.ragRuntime.DeleteMemory(memoryobserve.WithFields(ctx, map[string]any{
"trace_id": traceID,
}), documentIDs)
if err != nil {
s.observer.Observe(ctx, memoryobserve.Event{
Level: memoryobserve.LevelWarn,
Component: memoryobserve.ComponentWrite,
Operation: "vector_delete",
Fields: map[string]any{
"trace_id": traceID,
"item_count": len(memoryIDs),
"success": false,
"error": err,
"error_code": memoryobserve.ClassifyError(err),
},
})
for _, memoryID := range memoryIDs {
_ = s.itemRepo.UpdateVectorStateByID(ctx, memoryID, "failed", nil)
}
return
}
for _, memoryID := range memoryIDs {
_ = s.itemRepo.UpdateVectorStateByID(ctx, memoryID, "deleted", nil)
}
s.observer.Observe(ctx, memoryobserve.Event{
Level: memoryobserve.LevelInfo,
Component: memoryobserve.ComponentWrite,
Operation: "vector_delete",
Fields: map[string]any{
"trace_id": traceID,
"item_count": len(memoryIDs),
"success": true,
},
})
}
func parseMemoryID(documentID string) int64 {
documentID = strings.TrimSpace(documentID)
if !strings.HasPrefix(documentID, "memory:") {
return 0
}
raw := strings.TrimPrefix(documentID, "memory:")
if strings.HasPrefix(raw, "uid:") {
return 0
}
var value int64
for _, ch := range raw {
if ch < '0' || ch > '9' {
return 0
}
value = value*10 + int64(ch-'0')
}
return value
}
func strPtrOrNil(v string) *string {
v = strings.TrimSpace(v)
if v == "" {
return nil
}
value := v
return &value
}
func strValue(v *string) string {
if v == nil {
return ""
}
return strings.TrimSpace(*v)
}

View File

@@ -0,0 +1,248 @@
package worker
import (
"context"
"fmt"
"strings"
"github.com/LoveLosita/smartflow/backend/model"
memoryrepo "github.com/LoveLosita/smartflow/backend/services/memory/internal/repo"
memoryutils "github.com/LoveLosita/smartflow/backend/services/memory/internal/utils"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
// ApplyActionOutcome 是单个决策动作的执行结果。
//
// 说明:
// 1. Action 记录本次执行的动作类型ADD/UPDATE/DELETE/NONE
// 2. OldItem 仅在 UPDATE/DELETE 时有值,用于审计 before 快照;
// 3. NewItem 仅在 ADD/UPDATE 时有值,用于审计 after 快照和向量同步;
// 4. NeedsSync 标记是否需要触发向量同步ADD 和 UPDATE 需要)。
type ApplyActionOutcome struct {
Action string
MemoryID int64
OldItem *model.MemoryItem // UPDATE/DELETE 时的 before 快照
NewItem *model.MemoryItem // ADD/UPDATE 时的 after 快照
NeedsSync bool // 是否需要向量同步
}
// ApplyFinalDecision 把汇总后的最终决策落为数据库动作。
//
// 职责边界:
// 1. 在调用方事务内执行,不做独立事务管理;
// 2. 负责写 memory_items + memory_audit_logs不负责 job 状态推进;
// 3. 所有动作的审计日志都由这里统一产出。
//
// 参数说明:
// - itemRepo/auditRepo 必须是事务绑定的实例WithTx 后的);
// - fact 是当前正在处理的标准化事实;
// - job/payload 提供写入所需的上下文user_id、conversation_id 等)。
func ApplyFinalDecision(
ctx context.Context,
itemRepo *memoryrepo.ItemRepo,
auditRepo *memoryrepo.AuditRepo,
decision memorymodel.FinalDecision,
fact memorymodel.NormalizedFact,
job *model.MemoryJob,
payload memorymodel.ExtractJobPayload,
) (*ApplyActionOutcome, error) {
switch decision.Action {
case memorymodel.DecisionActionAdd:
return applyAdd(ctx, itemRepo, auditRepo, fact, job, payload, decision.Reason)
case memorymodel.DecisionActionUpdate:
return applyUpdate(ctx, itemRepo, auditRepo, decision, fact, job, payload)
case memorymodel.DecisionActionDelete:
return applyDelete(ctx, itemRepo, auditRepo, decision, payload.UserID)
case memorymodel.DecisionActionNone:
return &ApplyActionOutcome{
Action: memorymodel.DecisionActionNone,
NeedsSync: false,
}, nil
default:
return nil, fmt.Errorf("未知的决策动作: %s", decision.Action)
}
}
// applyAdd 执行新增动作:构建 MemoryItem → 写库 → 写审计。
func applyAdd(
ctx context.Context,
itemRepo *memoryrepo.ItemRepo,
auditRepo *memoryrepo.AuditRepo,
fact memorymodel.NormalizedFact,
job *model.MemoryJob,
payload memorymodel.ExtractJobPayload,
reason string,
) (*ApplyActionOutcome, error) {
// 1. 复用 runner.go 的 buildMemoryItems 构建单条 MemoryItem。
items := buildMemoryItems(job, payload, []memorymodel.NormalizedFact{fact})
if len(items) == 0 {
return nil, fmt.Errorf("构建记忆条目失败: memory_type=%s", fact.MemoryType)
}
// 2. 写库GORM Create 会自动填充 items[0].ID。
if err := itemRepo.UpsertItems(ctx, items); err != nil {
return nil, fmt.Errorf("新增记忆写入失败: %w", err)
}
// 注意:必须在 UpsertItems 之后取 items[0],因为 GORM Create 回填 ID 到 items[i]
// 之前用 item := items[0] 在 UpsertItems 之前拷贝,导致副本 ID 永远为 0。
item := items[0]
// 3. 写审计日志create 动作只有 after 快照)。
audit := memoryutils.BuildItemAuditLog(
item.ID,
item.UserID,
memoryutils.AuditOperationCreate,
"system",
formatAuditReason("决策层新增", reason),
nil,
&item,
)
if err := auditRepo.Create(ctx, audit); err != nil {
return nil, fmt.Errorf("新增审计写入失败: %w", err)
}
return &ApplyActionOutcome{
Action: memorymodel.DecisionActionAdd,
MemoryID: item.ID,
NewItem: &item,
NeedsSync: true,
}, nil
}
// applyUpdate 执行更新动作:查 before → 更新字段 → 写审计before+after
func applyUpdate(
ctx context.Context,
itemRepo *memoryrepo.ItemRepo,
auditRepo *memoryrepo.AuditRepo,
decision memorymodel.FinalDecision,
fact memorymodel.NormalizedFact,
job *model.MemoryJob,
payload memorymodel.ExtractJobPayload,
) (*ApplyActionOutcome, error) {
// 1. 查 before 快照,同时确认旧记忆存在且属于该用户。
oldItem, err := itemRepo.GetByIDForUser(ctx, payload.UserID, decision.TargetID)
if err != nil {
return nil, fmt.Errorf("查询旧记忆失败(id=%d): %w", decision.TargetID, err)
}
// 2. 重新计算 NormalizedContent 和 ContentHash保证和 NormalizeFacts 的逻辑一致。
// 原因LLM 输出的 merged content 需要重新走归一化链,避免大小写/空格差异导致后续 Hash 去重失效。
updatedContent := strings.TrimSpace(decision.Content)
if updatedContent == "" {
updatedContent = fact.Content
}
normalizedContent := strings.ToLower(updatedContent)
// 复用 utils.HashContent 的 sha256(memoryType + "::" + normalizedContent) 算法。
contentHash := memoryutils.HashContent(fact.MemoryType, normalizedContent)
title := strings.TrimSpace(decision.Title)
if title == "" {
title = oldItem.Title
}
// 3. 执行内容更新。
fields := memorymodel.UpdateContentFields{
Title: title,
Content: updatedContent,
NormalizedContent: normalizedContent,
ContentHash: contentHash,
Confidence: fact.Confidence,
Importance: fact.Importance,
}
if err := itemRepo.UpdateContentByID(ctx, decision.TargetID, fields); err != nil {
return nil, fmt.Errorf("更新记忆内容失败(id=%d): %w", decision.TargetID, err)
}
// 4. 构造 after 快照用于审计。
afterItem := *oldItem
afterItem.Title = title
afterItem.Content = updatedContent
if afterItem.NormalizedContent != nil {
afterItem.NormalizedContent = &normalizedContent
} else {
afterItem.NormalizedContent = strPtrFromValue(normalizedContent)
}
if afterItem.ContentHash != nil {
afterItem.ContentHash = &contentHash
} else {
afterItem.ContentHash = strPtrFromValue(contentHash)
}
afterItem.Confidence = fact.Confidence
afterItem.Importance = fact.Importance
// 5. 写审计日志update 动作同时有 before 和 after 快照)。
audit := memoryutils.BuildItemAuditLog(
oldItem.ID,
oldItem.UserID,
memoryutils.AuditOperationUpdate,
"system",
formatAuditReason("决策层更新", decision.Reason),
oldItem,
&afterItem,
)
if err := auditRepo.Create(ctx, audit); err != nil {
return nil, fmt.Errorf("更新审计写入失败: %w", err)
}
// 6. 向量状态重置为 pending触发向量重同步。
// 原因:内容变了,旧向量已过期,需要重新 embed。
_ = itemRepo.UpdateVectorStateByID(ctx, oldItem.ID, "pending", nil)
return &ApplyActionOutcome{
Action: memorymodel.DecisionActionUpdate,
MemoryID: oldItem.ID,
OldItem: oldItem,
NewItem: &afterItem,
NeedsSync: true,
}, nil
}
// applyDelete 执行软删除动作:查 before → 软删 → 写审计before only
func applyDelete(
ctx context.Context,
itemRepo *memoryrepo.ItemRepo,
auditRepo *memoryrepo.AuditRepo,
decision memorymodel.FinalDecision,
userID int,
) (*ApplyActionOutcome, error) {
// 1. 查 before 快照。
oldItem, err := itemRepo.GetByIDForUser(ctx, userID, decision.TargetID)
if err != nil {
return nil, fmt.Errorf("查询旧记忆失败(id=%d): %w", decision.TargetID, err)
}
// 2. 执行软删除。
if err := itemRepo.SoftDeleteByID(ctx, userID, decision.TargetID); err != nil {
return nil, fmt.Errorf("软删除记忆失败(id=%d): %w", decision.TargetID, err)
}
// 3. 写审计日志delete 动作只有 before 快照)。
audit := memoryutils.BuildItemAuditLog(
oldItem.ID,
oldItem.UserID,
memoryutils.AuditOperationDelete,
"system",
formatAuditReason("决策层删除", decision.Reason),
oldItem,
nil,
)
if err := auditRepo.Create(ctx, audit); err != nil {
return nil, fmt.Errorf("删除审计写入失败: %w", err)
}
return &ApplyActionOutcome{
Action: memorymodel.DecisionActionDelete,
MemoryID: oldItem.ID,
OldItem: oldItem,
NeedsSync: false,
}, nil
}
// formatAuditReason 统一审计日志的 reason 格式。
func formatAuditReason(prefix, detail string) string {
detail = strings.TrimSpace(detail)
if detail == "" {
return prefix
}
return prefix + ": " + detail
}

View File

@@ -0,0 +1,386 @@
package worker
import (
"context"
"fmt"
"github.com/LoveLosita/smartflow/backend/model"
memoryrepo "github.com/LoveLosita/smartflow/backend/services/memory/internal/repo"
memoryutils "github.com/LoveLosita/smartflow/backend/services/memory/internal/utils"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
ragservice "github.com/LoveLosita/smartflow/backend/services/rag"
"gorm.io/gorm"
)
// DecisionFlowOutcome 是一轮决策流程的汇总结果。
//
// 说明:
// 1. AddCount/UpdateCount/DeleteCount/NoneCount 分别统计四种动作的执行次数;
// 2. ItemsToSync 收集所有需要向量同步的 itemADD 和 UPDATE 产出的);
// 3. VectorDeletes 收集所有需要从向量库删除的 memory_idDELETE 动作产出的)。
type DecisionFlowOutcome struct {
AddCount int
UpdateCount int
DeleteCount int
NoneCount int
ItemsToSync []model.MemoryItem // 需要向量同步的新增/更新 item
VectorDeletes []int64 // 需要从向量库删除的 memory_id 列表
}
// factDecisionResult 是单条 fact 的决策执行结果,支持一对多动作。
// 原因conflict 场景下会产生 DELETE + ADD 两个动作,需要打包返回。
type factDecisionResult struct {
Outcomes []*ApplyActionOutcome
}
type candidateRecallResult struct {
Items []memorymodel.CandidateSnapshot
FallbackMode string
}
// executeDecisionFlow 在 worker 内编排"召回→逐对比对→汇总→执行"全流程。
//
// 职责边界:
// 1. 对每条 fact 独立执行完整决策流程fact 之间互不影响;
// 2. 所有数据库写操作在同一个事务内完成,保证原子性;
// 3. 向量同步在事务外异步执行,不影响事务提交。
//
// 降级策略:
// 1. Milvus 不可用时,回退到 MySQL 按类型查最近 N 条活跃记忆;
// 2. 单条 LLM 比对失败不影响其他候选,视为 unrelated
// 3. 整体流程报错时,由上层根据 FallbackMode 决定是否退回旧路径。
func (r *Runner) executeDecisionFlow(
ctx context.Context,
job *model.MemoryJob,
payload memorymodel.ExtractJobPayload,
facts []memorymodel.NormalizedFact,
) (*DecisionFlowOutcome, error) {
outcome := &DecisionFlowOutcome{
ItemsToSync: make([]model.MemoryItem, 0, len(facts)),
VectorDeletes: make([]int64, 0),
}
// 1. 所有数据库写操作在同一个事务内完成。
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
itemRepo := r.itemRepo.WithTx(tx)
auditRepo := r.auditRepo.WithTx(tx)
jobRepo := r.jobRepo.WithTx(tx)
for _, fact := range facts {
// 2. 对每条 fact 执行完整决策流程。
result, err := r.executeDecisionForFact(ctx, itemRepo, auditRepo, fact, job, payload)
if err != nil {
// 单条 fact 决策失败不影响其他 fact记录日志后继续。
if r.logger != nil {
r.logger.Printf("[WARN][去重] 单条 fact 决策失败,跳过继续: job_id=%d user_id=%d memory_type=%s hash=%s err=%v", job.ID, payload.UserID, fact.MemoryType, fact.ContentHash, err)
}
continue
}
// 3. 汇总结果到全局 outcome。
for _, actionOutcome := range result.Outcomes {
r.collectActionOutcome(outcome, actionOutcome)
}
}
// 4. 事务内最后确认 job 成功。
return jobRepo.MarkSuccess(ctx, job.ID)
})
if err != nil {
return nil, err
}
return outcome, nil
}
// executeDecisionForFact 对单条 fact 执行完整决策流程。
//
// 步骤:
// 1. Hash 精确命中检查 — 已有完全相同内容则直接跳过;
// 2. Milvus 语义召回 — 从旧记忆中筛出 TopK 候选(含降级);
// 3. 逐对 LLM 比对 — 每次拿一条新 fact 和一条旧候选比对;
// 4. 确定性汇总 — 根据 LLM 比对结果确定 ADD/UPDATE/DELETE/NONE
// 5. 校验 + 执行 — 落为数据库动作 + 审计日志。
func (r *Runner) executeDecisionForFact(
ctx context.Context,
itemRepo *memoryrepo.ItemRepo,
auditRepo *memoryrepo.AuditRepo,
fact memorymodel.NormalizedFact,
job *model.MemoryJob,
payload memorymodel.ExtractJobPayload,
) (*factDecisionResult, error) {
result := &factDecisionResult{}
// Step 1: Hash 精确命中检查。
// 原因:如果已有完全相同内容的记忆,直接跳过,无需调 LLM。
existing, err := itemRepo.FindActiveByHash(ctx, payload.UserID, fact.ContentHash)
if err != nil {
if r.logger != nil {
r.logger.Printf("[WARN][去重] Hash 精确匹配查询失败: user_id=%d memory_type=%s hash=%s err=%v", payload.UserID, fact.MemoryType, fact.ContentHash, err)
}
}
if len(existing) > 0 {
r.recordDecisionObservation(ctx, job, payload, fact, 0, memorymodel.DecisionActionNone, "hash_exact", true, nil)
result.Outcomes = append(result.Outcomes, &ApplyActionOutcome{
Action: memorymodel.DecisionActionNone,
NeedsSync: false,
})
return result, nil
}
// Step 2: Milvus 语义召回(含降级)。
recallResult := r.recallCandidates(ctx, payload, fact)
candidates := recallResult.Items
// 打印召回候选详情,便于排查向量召回和阈值过滤效果。
if r.logger != nil {
r.logger.Printf("[DEBUG][去重] 语义召回候选: job_id=%d user_id=%d memory_type=%s candidate_count=%d",
job.ID, payload.UserID, fact.MemoryType, len(candidates))
for _, c := range candidates {
r.logger.Printf("[DEBUG][去重] 候选详情: memory_id=%d score=%.4f content=\"%s\"",
c.MemoryID, c.Score, truncateRunes(c.Content, 50))
}
}
// Step 3: 逐对 LLM 比对。
comparisons := r.compareWithCandidates(ctx, fact, candidates)
// Step 4: 确定性汇总。
decision := memoryutils.AggregateComparisons(fact, comparisons, candidates)
// 打印汇总决策结果,便于排查去重终态。
if r.logger != nil {
r.logger.Printf("[DEBUG][去重] 汇总决策: job_id=%d action=%s target_id=%d reason=\"%s\"",
job.ID, decision.Action, decision.TargetID, decision.Reason)
}
// Step 5: 校验 + 执行。
actionOutcome, err := ApplyFinalDecision(ctx, itemRepo, auditRepo, *decision, fact, job, payload)
if err != nil {
r.recordDecisionObservation(ctx, job, payload, fact, len(candidates), decision.Action, recallResult.FallbackMode, false, err)
return nil, fmt.Errorf("执行决策动作失败: %w", err)
}
result.Outcomes = append(result.Outcomes, actionOutcome)
r.recordDecisionObservation(ctx, job, payload, fact, len(candidates), decision.Action, recallResult.FallbackMode, true, nil)
// Step 6: conflict (DELETE) 后需要补一个 ADD 写入新 fact。
// 原因:旧记忆矛盾需删除,但新事实本身仍然有效,必须写入。
if decision.Action == memorymodel.DecisionActionDelete {
addDecision := memorymodel.FinalDecision{
Action: memorymodel.DecisionActionAdd,
Reason: "冲突旧记忆已删除,写入新事实",
}
addOutcome, addErr := ApplyFinalDecision(ctx, itemRepo, auditRepo, addDecision, fact, job, payload)
if addErr != nil {
if r.logger != nil {
r.logger.Printf("[WARN] 冲突后补增失败: memory_type=%s err=%v", fact.MemoryType, addErr)
}
} else if addOutcome != nil {
result.Outcomes = append(result.Outcomes, addOutcome)
}
}
return result, nil
}
// recallCandidates 从旧记忆中召回候选,先尝试 Milvus降级时用 MySQL。
func (r *Runner) recallCandidates(
ctx context.Context,
payload memorymodel.ExtractJobPayload,
fact memorymodel.NormalizedFact,
) candidateRecallResult {
// 1. 优先使用 Milvus 向量语义召回。
if r.ragRuntime != nil {
retrieveResult, err := r.ragRuntime.RetrieveMemory(ctx, ragservice.MemoryRetrieveRequest{
Query: fact.Content,
TopK: r.cfg.DecisionCandidateTopK,
Threshold: r.cfg.DecisionCandidateMinScore,
UserID: payload.UserID,
MemoryTypes: []string{fact.MemoryType},
Action: "search",
})
if err == nil && len(retrieveResult.Items) > 0 {
candidates := r.buildCandidatesFromRAG(retrieveResult.Items)
if len(candidates) > 0 {
return candidateRecallResult{
Items: candidates,
FallbackMode: "rag",
}
}
// RAG 返回了结果但 DocumentID 全部解析失败,降级到 MySQL。
if r.logger != nil {
r.logger.Printf("[WARN][去重] Milvus 返回 %d 条结果但 DocumentID 全部解析失败,降级到 MySQL: user_id=%d memory_type=%s", len(retrieveResult.Items), payload.UserID, fact.MemoryType)
}
}
if err != nil && r.logger != nil {
r.logger.Printf("[WARN][去重] Milvus 语义召回失败,降级到 MySQL: user_id=%d memory_type=%s topk=%d err=%v", payload.UserID, fact.MemoryType, r.cfg.DecisionCandidateTopK, err)
}
return candidateRecallResult{
Items: r.recallCandidatesFromMySQL(ctx, payload, fact),
FallbackMode: "rag_to_mysql",
}
}
// 2. 降级:按 user_id + memory_type + status=active 查最近 N 条。
return candidateRecallResult{
Items: r.recallCandidatesFromMySQL(ctx, payload, fact),
FallbackMode: "mysql_only",
}
}
// buildCandidatesFromRAG 从 RAG 检索结果构建候选快照列表。
//
// 步骤:
// 1. 从 DocumentID格式 memory:{id})解析出 mysql_id
// 2. 从 metadata 提取 title 和 memory_type
// 3. 跳过无法解析 DocumentID 的结果。
func (r *Runner) buildCandidatesFromRAG(hits []ragservice.RetrieveHit) []memorymodel.CandidateSnapshot {
candidates := make([]memorymodel.CandidateSnapshot, 0, len(hits))
for _, hit := range hits {
memoryID := parseMemoryID(hit.DocumentID)
if memoryID <= 0 {
if r.logger != nil {
r.logger.Printf("[WARN][去重] DocumentID 解析失败,跳过候选: document_id=%q", hit.DocumentID)
}
continue
}
candidates = append(candidates, memorymodel.CandidateSnapshot{
MemoryID: memoryID,
Title: asStringFromMap(hit.Metadata, "title"),
Content: hit.Text,
MemoryType: asStringFromMap(hit.Metadata, "memory_type"),
Score: hit.Score,
})
}
return candidates
}
// recallCandidatesFromMySQL 从 MySQL 查最近 N 条活跃记忆作为候选。
// 这是 Milvus 不可用时的降级方案。
func (r *Runner) recallCandidatesFromMySQL(
ctx context.Context,
payload memorymodel.ExtractJobPayload,
fact memorymodel.NormalizedFact,
) []memorymodel.CandidateSnapshot {
items, err := r.itemRepo.FindByQuery(ctx, memorymodel.ItemQuery{
UserID: payload.UserID,
MemoryTypes: []string{fact.MemoryType},
Statuses: []string{model.MemoryItemStatusActive},
Limit: r.cfg.DecisionCandidateTopK,
})
if err != nil {
if r.logger != nil {
r.logger.Printf("[WARN] MySQL 降级召回失败: err=%v", err)
}
return nil
}
candidates := make([]memorymodel.CandidateSnapshot, 0, len(items))
for _, item := range items {
candidates = append(candidates, memorymodel.CandidateSnapshot{
MemoryID: item.ID,
Title: item.Title,
Content: item.Content,
MemoryType: item.MemoryType,
Score: 0, // MySQL 降级无向量分数
})
}
return candidates
}
// compareWithCandidates 对每个候选逐一调 LLM 做关系判断。
//
// 说明:
// 1. LLM 调用失败时视为 unrelated不影响其他候选的比对
// 2. 对比对结果做校验,不合法的也视为 unrelated
// 3. 无候选或决策编排器为空时返回空切片,上层直接走 ADD 路径。
func (r *Runner) compareWithCandidates(
ctx context.Context,
fact memorymodel.NormalizedFact,
candidates []memorymodel.CandidateSnapshot,
) []memorymodel.ComparisonResult {
if r.decisionOrchestrator == nil || len(candidates) == 0 {
return nil
}
comparisons := make([]memorymodel.ComparisonResult, 0, len(candidates))
for _, candidate := range candidates {
compResult, err := r.decisionOrchestrator.Compare(ctx, fact, candidate)
if err != nil {
// LLM 调用失败 → 视为 unrelated不影响其他候选。
if r.logger != nil {
r.logger.Printf("[WARN][去重] LLM 逐对比较调用失败,视为 unrelated: candidate_id=%d memory_type=%s err=%v", candidate.MemoryID, fact.MemoryType, err)
}
continue
}
// 校验 LLM 输出合法性,不合法也跳过。
if validateErr := memoryutils.ValidateComparisonResult(compResult); validateErr != nil {
if r.logger != nil {
r.logger.Printf("[WARN][去重] LLM 比对结果校验不通过,视为 unrelated: candidate_id=%d memory_type=%s relation=%s err=%v", candidate.MemoryID, fact.MemoryType, compResult.Relation, validateErr)
}
continue
}
comparisons = append(comparisons, *compResult)
// 打印 LLM 比对结果,便于排查误判。
if r.logger != nil {
r.logger.Printf("[DEBUG][去重] LLM 比对结果: candidate_id=%d score=%.4f relation=%s reason=\"%s\" candidate_content=\"%s\"",
candidate.MemoryID, candidate.Score, compResult.Relation, compResult.Reason, truncateRunes(candidate.Content, 50))
}
}
return comparisons
}
// collectActionOutcome 汇总单个动作结果到全局 outcome。
func (r *Runner) collectActionOutcome(outcome *DecisionFlowOutcome, actionOutcome *ApplyActionOutcome) {
if actionOutcome == nil {
return
}
switch actionOutcome.Action {
case memorymodel.DecisionActionAdd:
outcome.AddCount++
if actionOutcome.NeedsSync && actionOutcome.NewItem != nil {
outcome.ItemsToSync = append(outcome.ItemsToSync, *actionOutcome.NewItem)
}
case memorymodel.DecisionActionUpdate:
outcome.UpdateCount++
if actionOutcome.NeedsSync && actionOutcome.NewItem != nil {
outcome.ItemsToSync = append(outcome.ItemsToSync, *actionOutcome.NewItem)
}
case memorymodel.DecisionActionDelete:
outcome.DeleteCount++
outcome.VectorDeletes = append(outcome.VectorDeletes, actionOutcome.MemoryID)
case memorymodel.DecisionActionNone:
outcome.NoneCount++
}
}
// asStringFromMap 从 metadata map 中安全提取字符串值。
func asStringFromMap(m map[string]any, key string) string {
if m == nil {
return ""
}
v, ok := m[key]
if !ok || v == nil {
return ""
}
return fmt.Sprintf("%v", v)
}
// truncateRunes 截取字符串前 n 个 rune超出则追加 "..."。
// 用途:日志内容预览,避免超长内容撑爆单行日志。
func truncateRunes(s string, n int) string {
runes := []rune(s)
if len(runes) <= n {
return s
}
if n <= 0 {
return ""
}
return string(runes[:n]) + "..."
}

View File

@@ -0,0 +1,56 @@
package worker
import (
"context"
"log"
"time"
)
// RunPollingLoop 持续轮询 memory_jobs把异步 worker 真正跑起来。
//
// 职责边界:
// 1. 这里只负责“循环 + 轮询频率 + 批量触发”;
// 2. 不负责抽取逻辑,也不负责落库逻辑;
// 3. 任意一次 RunOnce 报错时只打日志并继续下一轮,避免整个后台循环退出。
func RunPollingLoop(ctx context.Context, runner *Runner, pollEvery time.Duration, claimBatch int) {
if runner == nil {
return
}
if runner.logger == nil {
runner.logger = log.Default()
}
if pollEvery <= 0 {
pollEvery = 2 * time.Second
}
if claimBatch <= 0 {
claimBatch = 1
}
runBatch := func() {
for i := 0; i < claimBatch; i++ {
result, err := runner.RunOnce(ctx)
if err != nil {
runner.logger.Printf("memory worker loop run once failed: %v", err)
return
}
if result == nil || !result.Claimed {
return
}
}
}
runBatch()
ticker := time.NewTicker(pollEvery)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
runner.logger.Printf("memory worker loop stopped: %v", ctx.Err())
return
case <-ticker.C:
runBatch()
}
}
}

View File

@@ -0,0 +1,22 @@
package worker
import (
"context"
memoryorchestrator "github.com/LoveLosita/smartflow/backend/services/memory/internal/orchestrator"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
)
// Extractor 是 worker 抽取依赖接口。
//
// 设计目的:
// 1. Day1 先接 mock 编排器跑通状态机;
// 2. Day2/Day3 可无缝替换为真实 LLM 抽取实现。
type Extractor interface {
ExtractFacts(ctx context.Context, payload memorymodel.ExtractJobPayload) ([]memorymodel.NormalizedFact, error)
}
// NewMockExtractor 返回 Day1 默认 mock 抽取器。
func NewMockExtractor() Extractor {
return memoryorchestrator.NewWriteOrchestrator()
}

View File

@@ -0,0 +1,483 @@
package worker
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"strings"
"time"
"github.com/LoveLosita/smartflow/backend/model"
memoryorchestrator "github.com/LoveLosita/smartflow/backend/services/memory/internal/orchestrator"
memoryrepo "github.com/LoveLosita/smartflow/backend/services/memory/internal/repo"
memoryutils "github.com/LoveLosita/smartflow/backend/services/memory/internal/utils"
memoryvectorsync "github.com/LoveLosita/smartflow/backend/services/memory/internal/vectorsync"
memorymodel "github.com/LoveLosita/smartflow/backend/services/memory/model"
memoryobserve "github.com/LoveLosita/smartflow/backend/services/memory/observe"
ragservice "github.com/LoveLosita/smartflow/backend/services/rag"
"gorm.io/gorm"
)
// RunOnceResult 描述单次手工触发执行的结果。
type RunOnceResult struct {
Claimed bool
JobID int64
Status string
Facts int
}
// Runner 负责把 memory_jobs 推进成 memory_items 和审计日志。
//
// 职责边界:
// 1. 负责任务抢占、抽取、落库和状态推进;
// 2. 不负责 outbox 消费,也不负责 LLM prompt 组装;
// 3. 失败时只做可恢复的状态回写,避免把业务错误直接抛到启动层。
type Runner struct {
db *gorm.DB
jobRepo *memoryrepo.JobRepo
itemRepo *memoryrepo.ItemRepo
auditRepo *memoryrepo.AuditRepo
settingsRepo *memoryrepo.SettingsRepo
extractor Extractor
ragRuntime ragservice.Runtime
logger *log.Logger
vectorSyncer *memoryvectorsync.Syncer
observer memoryobserve.Observer
metrics memoryobserve.MetricsRecorder
// 决策层依赖。
// 说明:
// 1. cfg 提供决策层配置是否启用、TopK、MinScore、FallbackMode
// 2. decisionOrchestrator 在决策启用时负责 LLM 逐对比较,为 nil 时走旧路径。
cfg memorymodel.Config
decisionOrchestrator *memoryorchestrator.LLMDecisionOrchestrator
}
// NewRunner 构造记忆 worker 执行器。
func NewRunner(
db *gorm.DB,
jobRepo *memoryrepo.JobRepo,
itemRepo *memoryrepo.ItemRepo,
auditRepo *memoryrepo.AuditRepo,
settingsRepo *memoryrepo.SettingsRepo,
extractor Extractor,
ragRuntime ragservice.Runtime,
cfg memorymodel.Config,
decisionOrchestrator *memoryorchestrator.LLMDecisionOrchestrator,
vectorSyncer *memoryvectorsync.Syncer,
observer memoryobserve.Observer,
metrics memoryobserve.MetricsRecorder,
) *Runner {
if observer == nil {
observer = memoryobserve.NewNopObserver()
}
if metrics == nil {
metrics = memoryobserve.NewNopMetrics()
}
return &Runner{
db: db,
jobRepo: jobRepo,
itemRepo: itemRepo,
auditRepo: auditRepo,
settingsRepo: settingsRepo,
extractor: extractor,
ragRuntime: ragRuntime,
logger: log.Default(),
vectorSyncer: vectorSyncer,
observer: observer,
metrics: metrics,
cfg: cfg,
decisionOrchestrator: decisionOrchestrator,
}
}
// RunOnce 手工执行一轮任务处理。
//
// 返回语义:
// 1. Claimed=false 表示当前没有可执行任务;
// 2. Claimed=true 且 Status=success/failed/dead 表示本轮已经推进过一个任务;
// 3. 只有初始化缺失或数据库级错误才返回 error。
func (r *Runner) RunOnce(ctx context.Context) (*RunOnceResult, error) {
if r == nil || r.db == nil || r.jobRepo == nil || r.itemRepo == nil || r.auditRepo == nil || r.settingsRepo == nil || r.extractor == nil {
return nil, errors.New("memory worker runner is not initialized")
}
// 1. 先抢占一条可执行任务,避免多个 worker 重复处理同一条记录。
job, err := r.jobRepo.ClaimNextRunnableExtractJob(ctx, time.Now())
if err != nil {
return nil, err
}
if job == nil {
return &RunOnceResult{Claimed: false}, nil
}
if job.RetryCount > 0 {
r.metrics.AddCounter(memoryobserve.MetricJobRetryTotal, 1, map[string]string{
"job_type": strings.TrimSpace(job.JobType),
})
}
result := &RunOnceResult{
Claimed: true,
JobID: job.ID,
Status: model.MemoryJobStatusProcessing,
Facts: 0,
}
// 2. 解析任务载荷。这里属于数据质量问题,解析失败就直接标记为可重试失败。
var payload memorymodel.ExtractJobPayload
if err = json.Unmarshal([]byte(job.PayloadJSON), &payload); err != nil {
failReason := fmt.Sprintf("解析任务载荷失败: %v", err)
_ = r.jobRepo.MarkFailed(ctx, job.ID, failReason)
result.Status = model.MemoryJobStatusFailed
r.recordJobOutcome(ctx, job, nil, result.Status, false, err)
return result, nil
}
// 3. 先读取用户记忆设置。总开关关闭时,任务直接成功结束,不再继续抽取和落库。
setting, err := r.settingsRepo.GetByUserID(ctx, payload.UserID)
if err != nil {
r.recordJobOutcome(ctx, job, &payload, model.MemoryJobStatusFailed, false, err)
return nil, err
}
effectiveSetting := memoryutils.EffectiveUserSetting(setting, payload.UserID)
if !effectiveSetting.MemoryEnabled {
if err = r.jobRepo.MarkSuccess(ctx, job.ID); err != nil {
r.recordJobOutcome(ctx, job, &payload, model.MemoryJobStatusFailed, false, err)
return nil, err
}
result.Status = model.MemoryJobStatusSuccess
r.logger.Printf("memory worker skipped by user setting: job_id=%d user_id=%d", job.ID, payload.UserID)
r.recordJobOutcome(ctx, job, &payload, result.Status, true, nil)
return result, nil
}
// 4. 调用抽取器。LLM 失败时由编排器做保守 fallbackworker 只关心最终结果。
facts, extractErr := r.extractor.ExtractFacts(ctx, payload)
if extractErr != nil {
failReason := fmt.Sprintf("抽取执行失败: %v", extractErr)
_ = r.jobRepo.MarkFailed(ctx, job.ID, failReason)
result.Status = model.MemoryJobStatusFailed
r.recordJobOutcome(ctx, job, &payload, result.Status, false, extractErr)
return result, nil
}
facts = memoryutils.FilterFactsBySetting(facts, effectiveSetting)
facts = memoryutils.FilterFactsByConfidence(facts, r.cfg.WriteMinConfidence)
if len(facts) == 0 {
if err = r.jobRepo.MarkSuccess(ctx, job.ID); err != nil {
r.recordJobOutcome(ctx, job, &payload, model.MemoryJobStatusFailed, false, err)
return nil, err
}
result.Status = model.MemoryJobStatusSuccess
r.logger.Printf("memory worker run once noop: job_id=%d", job.ID)
r.recordJobOutcome(ctx, job, &payload, result.Status, true, nil)
return result, nil
}
items := buildMemoryItems(job, payload, facts)
if len(items) == 0 {
if err = r.jobRepo.MarkSuccess(ctx, job.ID); err != nil {
r.recordJobOutcome(ctx, job, &payload, model.MemoryJobStatusFailed, false, err)
return nil, err
}
result.Status = model.MemoryJobStatusSuccess
r.logger.Printf("memory worker run once empty-after-normalize: job_id=%d", job.ID)
r.recordJobOutcome(ctx, job, &payload, result.Status, true, nil)
return result, nil
}
// 5. 根据配置选择写入路径:决策层 or 旧路径。
if r.cfg.DecisionEnabled && r.decisionOrchestrator != nil {
// 5a. 决策路径:召回→比对→汇总→执行。
outcome, decisionErr := r.executeDecisionFlow(ctx, job, payload, facts)
if decisionErr != nil {
// 决策流程整体失败,根据 FallbackMode 决定是否退回旧路径。
r.logger.Printf("[WARN][去重] 决策流程整体失败: job_id=%d user_id=%d facts_count=%d fallback=%s err=%v", job.ID, payload.UserID, len(facts), r.cfg.DecisionFallbackMode, decisionErr)
if r.cfg.DecisionFallbackMode == "legacy_add" {
if err = r.persistMemoryWrite(ctx, job.ID, items); err != nil {
failReason := fmt.Sprintf("决策降级后记忆落库失败: %v", err)
_ = r.jobRepo.MarkFailed(ctx, job.ID, failReason)
result.Status = model.MemoryJobStatusFailed
r.recordJobOutcome(ctx, job, &payload, result.Status, false, err)
return result, nil
}
result.Status = model.MemoryJobStatusSuccess
result.Facts = len(items)
r.syncMemoryVectors(ctx, items)
r.recordJobOutcome(ctx, job, &payload, result.Status, true, nil)
return result, nil
}
// FallbackMode=drop丢弃本轮抽取结果直接标记 job 成功。
_ = r.jobRepo.MarkSuccess(ctx, job.ID)
result.Status = model.MemoryJobStatusSuccess
r.recordJobOutcome(ctx, job, &payload, result.Status, true, nil)
return result, nil
}
// 5b. 决策成功:同步向量(新增/更新)和删除过期向量。
result.Status = model.MemoryJobStatusSuccess
result.Facts = outcome.AddCount + outcome.UpdateCount + outcome.DeleteCount
r.syncMemoryVectors(ctx, outcome.ItemsToSync)
r.syncVectorDeletes(ctx, outcome.VectorDeletes)
r.logger.Printf("[去重] 决策流程完成: job_id=%d user_id=%d 新增=%d 更新=%d 删除=%d 跳过=%d",
job.ID, payload.UserID, outcome.AddCount, outcome.UpdateCount, outcome.DeleteCount, outcome.NoneCount)
r.recordJobOutcome(ctx, job, &payload, result.Status, true, nil)
return result, nil
}
// 5c. 旧路径:和现在完全一样 — 先在事务里写入记忆条目和审计日志,再统一确认 job 成功。
if err = r.persistMemoryWrite(ctx, job.ID, items); err != nil {
failReason := fmt.Sprintf("记忆落库失败: %v", err)
_ = r.jobRepo.MarkFailed(ctx, job.ID, failReason)
result.Status = model.MemoryJobStatusFailed
r.recordJobOutcome(ctx, job, &payload, result.Status, false, err)
return result, nil
}
result.Status = model.MemoryJobStatusSuccess
result.Facts = len(items)
r.syncMemoryVectors(ctx, items)
r.logger.Printf("memory worker run once success: job_id=%d extracted_facts=%d", job.ID, len(items))
r.recordJobOutcome(ctx, job, &payload, result.Status, true, nil)
return result, nil
}
func (r *Runner) persistMemoryWrite(ctx context.Context, jobID int64, items []model.MemoryItem) error {
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
jobRepo := r.jobRepo.WithTx(tx)
itemRepo := r.itemRepo.WithTx(tx)
auditRepo := r.auditRepo.WithTx(tx)
if err := itemRepo.UpsertItems(ctx, items); err != nil {
return err
}
for i := range items {
audit := memoryutils.BuildItemAuditLog(
items[i].ID,
items[i].UserID,
memoryutils.AuditOperationCreate,
"system",
"LLM 提取入库",
nil,
&items[i],
)
if err := auditRepo.Create(ctx, audit); err != nil {
return err
}
}
return jobRepo.MarkSuccess(ctx, jobID)
})
}
func buildMemoryItems(job *model.MemoryJob, payload memorymodel.ExtractJobPayload, facts []memorymodel.NormalizedFact) []model.MemoryItem {
if job == nil || len(facts) == 0 {
return nil
}
items := make([]model.MemoryItem, 0, len(facts))
for _, fact := range facts {
items = append(items, model.MemoryItem{
UserID: payload.UserID,
ConversationID: strPtrOrNil(payload.ConversationID),
AssistantID: strPtrOrNil(payload.AssistantID),
RunID: strPtrOrNil(payload.RunID),
MemoryType: fact.MemoryType,
Title: fact.Title,
Content: fact.Content,
NormalizedContent: strPtrFromValue(fact.NormalizedContent),
ContentHash: strPtrFromValue(fact.ContentHash),
Confidence: fact.Confidence,
Importance: fact.Importance,
SensitivityLevel: fact.SensitivityLevel,
SourceMessageID: int64PtrOrNil(payload.SourceMessageID),
SourceEventID: job.SourceEventID,
IsExplicit: fact.IsExplicit,
Status: model.MemoryItemStatusActive,
TTLAt: resolveMemoryTTLAt(payload.OccurredAt, fact.MemoryType),
VectorStatus: "pending",
})
}
return items
}
func (r *Runner) syncMemoryVectors(ctx context.Context, items []model.MemoryItem) {
if r == nil || r.vectorSyncer == nil || len(items) == 0 {
return
}
r.vectorSyncer.Upsert(ctx, "", items)
}
// syncVectorDeletes 处理决策层 DELETE 动作产出的向量清理需求。
//
// 步骤:
// 1. 将 memoryID 转为 Milvus documentID"memory:{id}" 格式);
// 2. 调 Runtime.DeleteMemory 真正从 Milvus 删除对应向量;
// 3. 更新 MySQL vector_status 标记删除结果。
func (r *Runner) syncVectorDeletes(ctx context.Context, memoryIDs []int64) {
if r == nil || r.vectorSyncer == nil || len(memoryIDs) == 0 {
return
}
r.vectorSyncer.Delete(ctx, "", memoryIDs)
}
func resolveMemoryTTLAt(base time.Time, memoryType string) *time.Time {
switch memoryType {
case memorymodel.MemoryTypeFact:
t := base.Add(180 * 24 * time.Hour)
return &t
default:
return nil
}
}
func strPtrFromValue(v string) *string {
v = strings.TrimSpace(v)
if v == "" {
return nil
}
value := v
return &value
}
func strPtrOrNil(v string) *string {
v = strings.TrimSpace(v)
if v == "" {
return nil
}
value := v
return &value
}
func int64PtrOrNil(v int64) *int64 {
if v <= 0 {
return nil
}
value := v
return &value
}
func (r *Runner) recordJobOutcome(
ctx context.Context,
job *model.MemoryJob,
payload *memorymodel.ExtractJobPayload,
status string,
success bool,
err error,
) {
if r == nil {
return
}
level := memoryobserve.LevelInfo
if !success || err != nil {
level = memoryobserve.LevelWarn
}
fields := map[string]any{
"job_id": jobIDValue(job),
"status": strings.TrimSpace(status),
"success": success && err == nil,
"error": err,
"error_code": memoryobserve.ClassifyError(err),
}
if payload != nil {
fields["trace_id"] = strings.TrimSpace(payload.TraceID)
fields["user_id"] = payload.UserID
fields["conversation_id"] = strings.TrimSpace(payload.ConversationID)
}
r.observer.Observe(ctx, memoryobserve.Event{
Level: level,
Component: memoryobserve.ComponentWrite,
Operation: "job",
Fields: fields,
})
r.metrics.AddCounter(memoryobserve.MetricJobTotal, 1, map[string]string{
"status": strings.TrimSpace(status),
})
}
func (r *Runner) recordDecisionObservation(
ctx context.Context,
job *model.MemoryJob,
payload memorymodel.ExtractJobPayload,
fact memorymodel.NormalizedFact,
candidateCount int,
finalAction string,
fallbackMode string,
success bool,
err error,
) {
if r == nil {
return
}
level := memoryobserve.LevelInfo
status := "success"
if !success || err != nil {
level = memoryobserve.LevelWarn
status = "error"
}
fallbackMode = strings.TrimSpace(fallbackMode)
if fallbackMode == "" {
fallbackMode = "none"
}
r.observer.Observe(ctx, memoryobserve.Event{
Level: level,
Component: memoryobserve.ComponentWrite,
Operation: memoryobserve.OperationDecision,
Fields: map[string]any{
"trace_id": strings.TrimSpace(payload.TraceID),
"user_id": payload.UserID,
"conversation_id": strings.TrimSpace(payload.ConversationID),
"job_id": jobIDValue(job),
"fact_type": strings.TrimSpace(fact.MemoryType),
"candidate_count": candidateCount,
"final_action": strings.TrimSpace(finalAction),
"fallback_mode": fallbackMode,
"success": success && err == nil,
"error": err,
"error_code": memoryobserve.ClassifyError(err),
},
})
r.metrics.AddCounter(memoryobserve.MetricDecisionTotal, 1, map[string]string{
"action": strings.TrimSpace(finalAction),
"status": status,
})
if fallbackMode != "none" && fallbackMode != "hash_exact" && fallbackMode != "rag" {
r.metrics.AddCounter(memoryobserve.MetricDecisionFallbackTotal, 1, map[string]string{
"mode": fallbackMode,
})
}
}
func jobIDValue(job *model.MemoryJob) int64 {
if job == nil {
return 0
}
return job.ID
}
func parseMemoryID(documentID string) int64 {
documentID = strings.TrimSpace(documentID)
if !strings.HasPrefix(documentID, "memory:") {
return 0
}
raw := strings.TrimPrefix(documentID, "memory:")
if strings.HasPrefix(raw, "uid:") {
return 0
}
var value int64
for _, ch := range raw {
if ch < '0' || ch > '9' {
return 0
}
value = value*10 + int64(ch-'0')
}
return value
}