后端:
1. LLM 客户端从 newAgent/llm 提升为 infra/llm 基础设施层
- 删除 backend/newAgent/llm/(ark.go / ark_adapter.go / client.go / json.go)
- 等价迁移至 backend/infra/llm/,所有 newAgent node 与 service 统一改引用 infrallm
- 消除 newAgent 对模型客户端的私有依赖,为 memory / websearch 等多模块复用铺路
2. RAG 基础设施完成可运行态接入(factory / runtime / observer / service 四层成型)
- 新建 backend/infra/rag/factory.go / runtime.go / observe.go / observer.go /
service.go:工厂创建、运行时生命周期、轻量观测接口、检索服务门面
- 更新 infra/rag/config/config.go:补齐 Milvus / Embed / Reranker 全部配置项与默认值
- 更新 infra/rag/embed/eino_embedder.go:增强 Eino embedding 适配,支持 BaseURL / APIKey 环境变量 / 超时 /
维度等参数
- 更新 infra/rag/store/milvus_store.go:完整实现 Milvus 向量存储(建集合 / 建 Index / Upsert / Search /
Delete),支持 COSINE / L2 / IP 度量
- 更新 infra/rag/core/pipeline.go:适配 Runtime 接口,Pipeline 由 factory 注入而非手动拼装
- 更新 infra/rag/corpus/memory_corpus.go / vector_store.go:对接 Memory 模块数据源与 Store 接口扩展
3. Memory 模块从 Day1 骨架升级为 Day2 完整可运行态
- 新建 memory/module.go:统一门面 Module,对外封装 EnqueueExtract / ReadService / ManageService / WithTx /
StartWorker,启动层只依赖这一个入口
- 新建 memory/orchestrator/llm_write_orchestrator.go:LLM 驱动的记忆抽取编排器,替代原 mock 抽取
- 新建 memory/service/read_service.go:按用户开关过滤 + 轻量重排 + 访问时间刷新的读取链路
- 新建 memory/service/manage_service.go:记忆管理面能力(列出 / 软删除 / 开关读写),删除同步写审计日志
- 新建 memory/service/common.go:服务层公共工具
- 新建 memory/worker/loop.go:后台轮询循环 RunPollingLoop,定时抢占 pending 任务并推进
- 新建 memory/utils/audit.go / settings.go:审计日志构造、用户设置过滤等纯函数
- 更新 memory/model/item.go / job.go / settings.go / config.go / status.go:补齐 DTO 字段与状态常量
- 更新 memory/repo/item_repo.go / job_repo.go / audit_repo.go / settings_repo.go:补齐 CRUD 与查询能力
- 更新 memory/worker/runner.go:Runner 对接 Module 与 LLM 抽取器,任务状态机完整化
- 更新 memory/README.md:同步模块现状说明
4. newAgent 接入 Memory 读取注入与工具注册依赖预埋
- 新建 service/agentsvc/agent_memory.go:定义 MemoryReader 接口 + injectMemoryContext,在 graph
执行前统一补充记忆上下文
- 更新 service/agentsvc/agent.go:新增 memoryReader 字段与 SetMemoryReader 方法
- 更新 service/agentsvc/agent_newagent.go:调用 injectMemoryContext 注入 pinned block,检索失败仅降级不阻断主链路
- 更新 newAgent/tools/registry.go:新增 DefaultRegistryDeps(含 RAGRuntime),工具注册表支持依赖注入
5. 启动流程与事件处理器接线更新
- 更新 cmd/start.go:初始化 RAG Runtime → Memory Module → 注册事件处理器 → 启动 Worker 后台轮询
- 更新 service/events/memory_extract_requested.go:改用 memory.Module.WithTx(tx) 统一门面,事件处理器不再直接依赖
repo/service 内部包
6. 缓存插件与配置同步
- 更新 middleware/cache_deleter.go:静默忽略 MemoryJob / MemoryItem / MemoryAuditLog / MemoryUserSetting
等新模型,避免日志刷屏;清理冗余注释
- 更新 config.example.yaml:补齐 rag / memory / websearch 配置段及默认值
- 更新 go.mod / go.sum:新增 eino-ext/openai / json-patch / go-openai 依赖
前端:无 仓库:无
223 lines
5.5 KiB
Go
223 lines
5.5 KiB
Go
package repo
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"time"
|
||
|
||
memorymodel "github.com/LoveLosita/smartflow/backend/memory/model"
|
||
"github.com/LoveLosita/smartflow/backend/model"
|
||
"gorm.io/gorm"
|
||
"gorm.io/gorm/clause"
|
||
)
|
||
|
||
// JobRepo 封装 memory_jobs 的数据访问。
|
||
type JobRepo struct {
|
||
db *gorm.DB
|
||
}
|
||
|
||
func NewJobRepo(db *gorm.DB) *JobRepo {
|
||
return &JobRepo{db: db}
|
||
}
|
||
|
||
func (r *JobRepo) WithTx(tx *gorm.DB) *JobRepo {
|
||
return &JobRepo{db: tx}
|
||
}
|
||
|
||
// CreatePendingExtractJob 创建“待抽取”任务(幂等写入)。
|
||
//
|
||
// 失败语义:
|
||
// 1. 参数非法直接返回 error,由上游决定 dead 或重试;
|
||
// 2. 同幂等键重复写入采用 DoNothing,保证无副作用。
|
||
func (r *JobRepo) CreatePendingExtractJob(
|
||
ctx context.Context,
|
||
payload memorymodel.ExtractJobPayload,
|
||
sourceEventID string,
|
||
) error {
|
||
if r == nil || r.db == nil {
|
||
return errors.New("memory job repo is nil")
|
||
}
|
||
if payload.UserID <= 0 {
|
||
return errors.New("invalid user_id")
|
||
}
|
||
if payload.IdempotencyKey == "" {
|
||
return errors.New("idempotency_key is empty")
|
||
}
|
||
|
||
rawPayload, err := json.Marshal(payload)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
now := time.Now()
|
||
job := model.MemoryJob{
|
||
UserID: payload.UserID,
|
||
ConversationID: strPtrOrNil(payload.ConversationID),
|
||
SourceMessageID: int64PtrOrNil(payload.SourceMessageID),
|
||
SourceEventID: strPtrOrNil(sourceEventID),
|
||
JobType: model.MemoryJobTypeExtract,
|
||
IdempotencyKey: payload.IdempotencyKey,
|
||
PayloadJSON: string(rawPayload),
|
||
Status: model.MemoryJobStatusPending,
|
||
RetryCount: 0,
|
||
MaxRetry: 6,
|
||
NextRetryAt: &now,
|
||
}
|
||
|
||
return r.db.WithContext(ctx).
|
||
Clauses(clause.OnConflict{
|
||
Columns: []clause.Column{{Name: "idempotency_key"}},
|
||
DoNothing: true,
|
||
}).
|
||
Create(&job).Error
|
||
}
|
||
|
||
// ClaimNextRunnableExtractJob 抢占一个可执行的 extract 任务。
|
||
//
|
||
// 抢占规则:
|
||
// 1. 只从 pending/failed 中挑 next_retry_at 已到期任务;
|
||
// 2. 用行锁避免多个 worker 抢到同一条任务;
|
||
// 3. 抢占成功后立即置为 processing,防止重复执行。
|
||
func (r *JobRepo) ClaimNextRunnableExtractJob(ctx context.Context, now time.Time) (*model.MemoryJob, error) {
|
||
if r == nil || r.db == nil {
|
||
return nil, errors.New("memory job repo is nil")
|
||
}
|
||
|
||
var claimed *model.MemoryJob
|
||
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||
var job model.MemoryJob
|
||
query := tx.
|
||
Clauses(clause.Locking{Strength: "UPDATE"}).
|
||
Where("job_type = ?", model.MemoryJobTypeExtract).
|
||
Where("status IN ?", []string{model.MemoryJobStatusPending, model.MemoryJobStatusFailed}).
|
||
Where("(next_retry_at IS NULL OR next_retry_at <= ?)", now).
|
||
Order("id ASC").
|
||
Limit(1).
|
||
Find(&job)
|
||
if query.Error != nil {
|
||
return query.Error
|
||
}
|
||
if query.RowsAffected == 0 {
|
||
return nil
|
||
}
|
||
|
||
updates := map[string]any{
|
||
"status": model.MemoryJobStatusProcessing,
|
||
"updated_at": now,
|
||
"last_error": nil,
|
||
}
|
||
if updateErr := tx.Model(&model.MemoryJob{}).Where("id = ?", job.ID).Updates(updates).Error; updateErr != nil {
|
||
return updateErr
|
||
}
|
||
|
||
job.Status = model.MemoryJobStatusProcessing
|
||
job.UpdatedAt = &now
|
||
claimed = &job
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return claimed, nil
|
||
}
|
||
|
||
// MarkSuccess 把任务推进为 success 最终态。
|
||
func (r *JobRepo) MarkSuccess(ctx context.Context, jobID int64) error {
|
||
if r == nil || r.db == nil {
|
||
return errors.New("memory job repo is nil")
|
||
}
|
||
now := time.Now()
|
||
updates := map[string]any{
|
||
"status": model.MemoryJobStatusSuccess,
|
||
"last_error": nil,
|
||
"next_retry_at": nil,
|
||
"updated_at": now,
|
||
}
|
||
return r.db.WithContext(ctx).Model(&model.MemoryJob{}).Where("id = ?", jobID).Updates(updates).Error
|
||
}
|
||
|
||
// MarkFailed 按重试策略推进任务到 failed/dead。
|
||
//
|
||
// 规则:
|
||
// 1. retry_count +1 后若超上限,直接 dead;
|
||
// 2. 未超上限则写 failed 并设置 next_retry_at。
|
||
func (r *JobRepo) MarkFailed(ctx context.Context, jobID int64, reason string) error {
|
||
if r == nil || r.db == nil {
|
||
return errors.New("memory job repo is nil")
|
||
}
|
||
|
||
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||
var job model.MemoryJob
|
||
queryErr := tx.
|
||
Clauses(clause.Locking{Strength: "UPDATE"}).
|
||
Where("id = ?", jobID).
|
||
First(&job).Error
|
||
if queryErr != nil {
|
||
return queryErr
|
||
}
|
||
if job.Status == model.MemoryJobStatusSuccess || job.Status == model.MemoryJobStatusDead {
|
||
return nil
|
||
}
|
||
|
||
maxRetry := job.MaxRetry
|
||
if maxRetry <= 0 {
|
||
maxRetry = 6
|
||
}
|
||
nextRetryCount := job.RetryCount + 1
|
||
now := time.Now()
|
||
status := model.MemoryJobStatusFailed
|
||
var nextRetryAt *time.Time
|
||
if nextRetryCount >= maxRetry {
|
||
status = model.MemoryJobStatusDead
|
||
nextRetryAt = nil
|
||
} else {
|
||
t := now.Add(calcRetryBackoff(nextRetryCount))
|
||
nextRetryAt = &t
|
||
}
|
||
|
||
lastErr := truncateError(reason)
|
||
updates := map[string]any{
|
||
"status": status,
|
||
"retry_count": nextRetryCount,
|
||
"last_error": &lastErr,
|
||
"next_retry_at": nextRetryAt,
|
||
"updated_at": now,
|
||
}
|
||
return tx.Model(&model.MemoryJob{}).Where("id = ?", jobID).Updates(updates).Error
|
||
})
|
||
}
|
||
|
||
func calcRetryBackoff(retryCount int) time.Duration {
|
||
if retryCount <= 0 {
|
||
return time.Second
|
||
}
|
||
if retryCount > 6 {
|
||
retryCount = 6
|
||
}
|
||
return time.Second * time.Duration(1<<(retryCount-1))
|
||
}
|
||
|
||
func truncateError(reason string) string {
|
||
if len(reason) <= 2000 {
|
||
return reason
|
||
}
|
||
return reason[:2000]
|
||
}
|
||
|
||
func strPtrOrNil(v string) *string {
|
||
if v == "" {
|
||
return nil
|
||
}
|
||
value := v
|
||
return &value
|
||
}
|
||
|
||
func int64PtrOrNil(v int64) *int64 {
|
||
if v <= 0 {
|
||
return nil
|
||
}
|
||
value := v
|
||
return &value
|
||
}
|