Version: 0.9.13.dev.260410
后端： 1. Memory Day1 链路打通（chat_history -> outbox -> memory_jobs） - 更新 service/events/chat_history_persist.go：聊天消息落库同事务追加 memory.extract.requested 事件（仅 user 消息，失败回滚后由 outbox 重试） - 新建 service/events/memory_extract_requested.go：消费 memory.extract.requested 并幂等入队 memory_jobs，补齐 payload 校验、文本截断与 idempotency key - 更新 cmd/start.go：注册 RegisterMemoryExtractRequestedHandler 2. Memory 模块骨架落地（先跑通状态机，再接入真实抽取） - 新建 memory/model、repo、service、orchestrator、worker、utils 目录与 Day1 mock 抽取执行链 - 新建 model/memory.go：补齐 memory_items / memory_jobs / memory_audit_logs / memory_user_settings 与事件 payload 模型 - 更新 inits/mysql.go：接入 4 张 memory 相关表 AutoMigrate 3. RAG 复用基础设施预埋（依赖可替换） - 新建 infra/rag：core pipeline + chunk/embed/retrieve/rerank/store/corpus/config 分层实现 - 默认接入 MockEmbedder + InMemoryStore，预留 Milvus / Eino 适配实现 - 新增 infra/rag/RAG复用接口实施计划.md 4. 本地依赖与交接文档同步 - 更新 docker-compose.yml：新增 etcd / minio / milvus / attu 服务与数据卷 - 删除 newAgent/HANDOFF_工具研究与运行态重置.md、newAgent/阶段3_上下文瘦身设计.md - 新增 newAgent/HANDOFF_WebSearch两阶段实施计划.md、memory/HANDOFF-RAG复用后续实施计划.md、memory/README.md 前端：无 仓库：无
This commit is contained in:
LoveLosita
25
backend/memory/repo/audit_repo.go
Normal file
25
backend/memory/repo/audit_repo.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/LoveLosita/smartflow/backend/model"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// AuditRepo 封装 memory_audit_logs 的数据访问。
|
||||
type AuditRepo struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func NewAuditRepo(db *gorm.DB) *AuditRepo {
|
||||
return &AuditRepo{db: db}
|
||||
}
|
||||
|
||||
func (r *AuditRepo) Create(ctx context.Context, log model.MemoryAuditLog) error {
|
||||
if r == nil || r.db == nil {
|
||||
return errors.New("memory audit repo is nil")
|
||||
}
|
||||
return r.db.WithContext(ctx).Create(&log).Error
|
||||
}
|
||||
33
backend/memory/repo/item_repo.go
Normal file
33
backend/memory/repo/item_repo.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/LoveLosita/smartflow/backend/model"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// ItemRepo 封装 memory_items 的数据访问（Day1 先占位）。
|
||||
type ItemRepo struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func NewItemRepo(db *gorm.DB) *ItemRepo {
|
||||
return &ItemRepo{db: db}
|
||||
}
|
||||
|
||||
// UpsertItems 预留给 Day2/Day3 的写入链路。
|
||||
//
|
||||
// Day1 约束：
|
||||
// 1. 先完成任务入队与状态机闭环；
|
||||
// 2. 不在本阶段引入复杂冲突消解与向量写入。
|
||||
func (r *ItemRepo) UpsertItems(ctx context.Context, items []model.MemoryItem) error {
|
||||
if r == nil || r.db == nil {
|
||||
return errors.New("memory item repo is nil")
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
return r.db.WithContext(ctx).Create(&items).Error
|
||||
}
|
||||
221
backend/memory/repo/job_repo.go
Normal file
221
backend/memory/repo/job_repo.go
Normal file
@@ -0,0 +1,221 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
memorymodel "github.com/LoveLosita/smartflow/backend/memory/model"
|
||||
"github.com/LoveLosita/smartflow/backend/model"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// JobRepo 封装 memory_jobs 的数据访问。
|
||||
type JobRepo struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func NewJobRepo(db *gorm.DB) *JobRepo {
|
||||
return &JobRepo{db: db}
|
||||
}
|
||||
|
||||
func (r *JobRepo) WithTx(tx *gorm.DB) *JobRepo {
|
||||
return &JobRepo{db: tx}
|
||||
}
|
||||
|
||||
// CreatePendingExtractJob 创建“待抽取”任务（幂等写入）。
|
||||
//
|
||||
// 失败语义：
|
||||
// 1. 参数非法直接返回 error，由上游决定 dead 或重试；
|
||||
// 2. 同幂等键重复写入采用 DoNothing，保证无副作用。
|
||||
func (r *JobRepo) CreatePendingExtractJob(
|
||||
ctx context.Context,
|
||||
payload memorymodel.ExtractJobPayload,
|
||||
sourceEventID string,
|
||||
) error {
|
||||
if r == nil || r.db == nil {
|
||||
return errors.New("memory job repo is nil")
|
||||
}
|
||||
if payload.UserID <= 0 {
|
||||
return errors.New("invalid user_id")
|
||||
}
|
||||
if payload.IdempotencyKey == "" {
|
||||
return errors.New("idempotency_key is empty")
|
||||
}
|
||||
|
||||
rawPayload, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
job := model.MemoryJob{
|
||||
UserID: payload.UserID,
|
||||
ConversationID: strPtrOrNil(payload.ConversationID),
|
||||
SourceMessageID: int64PtrOrNil(payload.SourceMessageID),
|
||||
SourceEventID: strPtrOrNil(sourceEventID),
|
||||
JobType: model.MemoryJobTypeExtract,
|
||||
IdempotencyKey: payload.IdempotencyKey,
|
||||
PayloadJSON: string(rawPayload),
|
||||
Status: model.MemoryJobStatusPending,
|
||||
RetryCount: 0,
|
||||
MaxRetry: 6,
|
||||
NextRetryAt: &now,
|
||||
}
|
||||
|
||||
return r.db.WithContext(ctx).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "idempotency_key"}},
|
||||
DoNothing: true,
|
||||
}).
|
||||
Create(&job).Error
|
||||
}
|
||||
|
||||
// ClaimNextRunnableExtractJob 抢占一个可执行的 extract 任务。
|
||||
//
|
||||
// 抢占规则：
|
||||
// 1. 只从 pending/failed 中挑 next_retry_at 已到期任务；
|
||||
// 2. 用行锁避免多个 worker 抢到同一条任务；
|
||||
// 3. 抢占成功后立即置为 processing，防止重复执行。
|
||||
func (r *JobRepo) ClaimNextRunnableExtractJob(ctx context.Context, now time.Time) (*model.MemoryJob, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return nil, errors.New("memory job repo is nil")
|
||||
}
|
||||
|
||||
var claimed *model.MemoryJob
|
||||
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var job model.MemoryJob
|
||||
queryErr := tx.
|
||||
Clauses(clause.Locking{Strength: "UPDATE"}).
|
||||
Where("job_type = ?", model.MemoryJobTypeExtract).
|
||||
Where("status IN ?", []string{model.MemoryJobStatusPending, model.MemoryJobStatusFailed}).
|
||||
Where("(next_retry_at IS NULL OR next_retry_at <= ?)", now).
|
||||
Order("id ASC").
|
||||
First(&job).Error
|
||||
if queryErr != nil {
|
||||
if errors.Is(queryErr, gorm.ErrRecordNotFound) {
|
||||
return nil
|
||||
}
|
||||
return queryErr
|
||||
}
|
||||
|
||||
updates := map[string]any{
|
||||
"status": model.MemoryJobStatusProcessing,
|
||||
"updated_at": now,
|
||||
"last_error": nil,
|
||||
}
|
||||
if updateErr := tx.Model(&model.MemoryJob{}).Where("id = ?", job.ID).Updates(updates).Error; updateErr != nil {
|
||||
return updateErr
|
||||
}
|
||||
|
||||
job.Status = model.MemoryJobStatusProcessing
|
||||
job.UpdatedAt = &now
|
||||
claimed = &job
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return claimed, nil
|
||||
}
|
||||
|
||||
// MarkSuccess 把任务推进为 success 最终态。
|
||||
func (r *JobRepo) MarkSuccess(ctx context.Context, jobID int64) error {
|
||||
if r == nil || r.db == nil {
|
||||
return errors.New("memory job repo is nil")
|
||||
}
|
||||
now := time.Now()
|
||||
updates := map[string]any{
|
||||
"status": model.MemoryJobStatusSuccess,
|
||||
"last_error": nil,
|
||||
"next_retry_at": nil,
|
||||
"updated_at": now,
|
||||
}
|
||||
return r.db.WithContext(ctx).Model(&model.MemoryJob{}).Where("id = ?", jobID).Updates(updates).Error
|
||||
}
|
||||
|
||||
// MarkFailed 按重试策略推进任务到 failed/dead。
|
||||
//
|
||||
// 规则：
|
||||
// 1. retry_count +1 后若超上限，直接 dead；
|
||||
// 2. 未超上限则写 failed 并设置 next_retry_at。
|
||||
func (r *JobRepo) MarkFailed(ctx context.Context, jobID int64, reason string) error {
|
||||
if r == nil || r.db == nil {
|
||||
return errors.New("memory job repo is nil")
|
||||
}
|
||||
|
||||
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var job model.MemoryJob
|
||||
queryErr := tx.
|
||||
Clauses(clause.Locking{Strength: "UPDATE"}).
|
||||
Where("id = ?", jobID).
|
||||
First(&job).Error
|
||||
if queryErr != nil {
|
||||
return queryErr
|
||||
}
|
||||
if job.Status == model.MemoryJobStatusSuccess || job.Status == model.MemoryJobStatusDead {
|
||||
return nil
|
||||
}
|
||||
|
||||
maxRetry := job.MaxRetry
|
||||
if maxRetry <= 0 {
|
||||
maxRetry = 6
|
||||
}
|
||||
nextRetryCount := job.RetryCount + 1
|
||||
now := time.Now()
|
||||
status := model.MemoryJobStatusFailed
|
||||
var nextRetryAt *time.Time
|
||||
if nextRetryCount >= maxRetry {
|
||||
status = model.MemoryJobStatusDead
|
||||
nextRetryAt = nil
|
||||
} else {
|
||||
t := now.Add(calcRetryBackoff(nextRetryCount))
|
||||
nextRetryAt = &t
|
||||
}
|
||||
|
||||
lastErr := truncateError(reason)
|
||||
updates := map[string]any{
|
||||
"status": status,
|
||||
"retry_count": nextRetryCount,
|
||||
"last_error": &lastErr,
|
||||
"next_retry_at": nextRetryAt,
|
||||
"updated_at": now,
|
||||
}
|
||||
return tx.Model(&model.MemoryJob{}).Where("id = ?", jobID).Updates(updates).Error
|
||||
})
|
||||
}
|
||||
|
||||
func calcRetryBackoff(retryCount int) time.Duration {
|
||||
if retryCount <= 0 {
|
||||
return time.Second
|
||||
}
|
||||
if retryCount > 6 {
|
||||
retryCount = 6
|
||||
}
|
||||
return time.Second * time.Duration(1<<(retryCount-1))
|
||||
}
|
||||
|
||||
func truncateError(reason string) string {
|
||||
if len(reason) <= 2000 {
|
||||
return reason
|
||||
}
|
||||
return reason[:2000]
|
||||
}
|
||||
|
||||
func strPtrOrNil(v string) *string {
|
||||
if v == "" {
|
||||
return nil
|
||||
}
|
||||
value := v
|
||||
return &value
|
||||
}
|
||||
|
||||
func int64PtrOrNil(v int64) *int64 {
|
||||
if v <= 0 {
|
||||
return nil
|
||||
}
|
||||
value := v
|
||||
return &value
|
||||
}
|
||||
34
backend/memory/repo/settings_repo.go
Normal file
34
backend/memory/repo/settings_repo.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/LoveLosita/smartflow/backend/model"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// SettingsRepo 封装 memory_user_settings 的读写。
|
||||
type SettingsRepo struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func NewSettingsRepo(db *gorm.DB) *SettingsRepo {
|
||||
return &SettingsRepo{db: db}
|
||||
}
|
||||
|
||||
func (r *SettingsRepo) Upsert(ctx context.Context, setting model.MemoryUserSetting) error {
|
||||
if r == nil || r.db == nil {
|
||||
return errors.New("memory settings repo is nil")
|
||||
}
|
||||
return r.db.WithContext(ctx).Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "user_id"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{
|
||||
"memory_enabled",
|
||||
"implicit_memory_enabled",
|
||||
"sensitive_memory_enabled",
|
||||
"updated_at",
|
||||
}),
|
||||
}).Create(&setting).Error
|
||||
}
|
||||
Reference in New Issue
Block a user