Version: 0.9.13.dev.260410
后端: 1. Memory Day1 链路打通(chat_history -> outbox -> memory_jobs) - 更新 service/events/chat_history_persist.go:聊天消息落库同事务追加 memory.extract.requested 事件(仅 user 消息,失败回滚后由 outbox 重试) - 新建 service/events/memory_extract_requested.go:消费 memory.extract.requested 并幂等入队 memory_jobs,补齐 payload 校验、文本截断与 idempotency key - 更新 cmd/start.go:注册 RegisterMemoryExtractRequestedHandler 2. Memory 模块骨架落地(先跑通状态机,再接入真实抽取) - 新建 memory/model、repo、service、orchestrator、worker、utils 目录与 Day1 mock 抽取执行链 - 新建 model/memory.go:补齐 memory_items / memory_jobs / memory_audit_logs / memory_user_settings 与事件 payload 模型 - 更新 inits/mysql.go:接入 4 张 memory 相关表 AutoMigrate 3. RAG 复用基础设施预埋(依赖可替换) - 新建 infra/rag:core pipeline + chunk/embed/retrieve/rerank/store/corpus/config 分层实现 - 默认接入 MockEmbedder + InMemoryStore,预留 Milvus / Eino 适配实现 - 新增 infra/rag/RAG复用接口实施计划.md 4. 本地依赖与交接文档同步 - 更新 docker-compose.yml:新增 etcd / minio / milvus / attu 服务与数据卷 - 删除 newAgent/HANDOFF_工具研究与运行态重置.md、newAgent/阶段3_上下文瘦身设计.md - 新增 newAgent/HANDOFF_WebSearch两阶段实施计划.md、memory/HANDOFF-RAG复用后续实施计划.md、memory/README.md 前端:无 仓库:无
This commit is contained in:
166
backend/infra/rag/store/inmemory_store.go
Normal file
166
backend/infra/rag/store/inmemory_store.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/LoveLosita/smartflow/backend/infra/rag/core"
|
||||
)
|
||||
|
||||
// InMemoryVectorStore 是本地开发用向量存储实现。
|
||||
//
|
||||
// 注意:
|
||||
// 1. 仅用于开发调试,不建议生产使用;
|
||||
// 2. 真实环境可替换为 MilvusStore,接口保持一致。
|
||||
type InMemoryVectorStore struct {
|
||||
mu sync.RWMutex
|
||||
rows map[string]core.VectorRow
|
||||
}
|
||||
|
||||
func NewInMemoryVectorStore() *InMemoryVectorStore {
|
||||
return &InMemoryVectorStore{
|
||||
rows: make(map[string]core.VectorRow),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *InMemoryVectorStore) Upsert(_ context.Context, rows []core.VectorRow) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
now := time.Now()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, row := range rows {
|
||||
current, exists := s.rows[row.ID]
|
||||
if exists {
|
||||
row.CreatedAt = current.CreatedAt
|
||||
row.UpdatedAt = now
|
||||
} else {
|
||||
if row.CreatedAt.IsZero() {
|
||||
row.CreatedAt = now
|
||||
}
|
||||
row.UpdatedAt = now
|
||||
}
|
||||
s.rows[row.ID] = row
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *InMemoryVectorStore) Search(_ context.Context, req core.VectorSearchRequest) ([]core.ScoredVectorRow, error) {
|
||||
topK := req.TopK
|
||||
if topK <= 0 {
|
||||
topK = 8
|
||||
}
|
||||
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
result := make([]core.ScoredVectorRow, 0, len(s.rows))
|
||||
for _, row := range s.rows {
|
||||
if !matchMetadataFilter(row.Metadata, req.Filter) {
|
||||
continue
|
||||
}
|
||||
score := cosineSimilarity(req.QueryVector, row.Vector)
|
||||
result = append(result, core.ScoredVectorRow{
|
||||
Row: row,
|
||||
Score: score,
|
||||
})
|
||||
}
|
||||
sort.SliceStable(result, func(i, j int) bool {
|
||||
return result[i].Score > result[j].Score
|
||||
})
|
||||
if len(result) <= topK {
|
||||
return result, nil
|
||||
}
|
||||
return result[:topK], nil
|
||||
}
|
||||
|
||||
func (s *InMemoryVectorStore) Delete(_ context.Context, ids []string) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, id := range ids {
|
||||
delete(s.rows, id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *InMemoryVectorStore) Get(_ context.Context, ids []string) ([]core.VectorRow, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
result := make([]core.VectorRow, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
row, exists := s.rows[id]
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
result = append(result, row)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func cosineSimilarity(a, b []float32) float64 {
|
||||
if len(a) == 0 || len(b) == 0 {
|
||||
return 0
|
||||
}
|
||||
n := len(a)
|
||||
if len(b) < n {
|
||||
n = len(b)
|
||||
}
|
||||
if n == 0 {
|
||||
return 0
|
||||
}
|
||||
var dot, normA, normB float64
|
||||
for i := 0; i < n; i++ {
|
||||
av := float64(a[i])
|
||||
bv := float64(b[i])
|
||||
dot += av * bv
|
||||
normA += av * av
|
||||
normB += bv * bv
|
||||
}
|
||||
if normA == 0 || normB == 0 {
|
||||
return 0
|
||||
}
|
||||
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
|
||||
}
|
||||
|
||||
func matchMetadataFilter(metadata map[string]any, filter map[string]any) bool {
|
||||
if len(filter) == 0 {
|
||||
return true
|
||||
}
|
||||
for key, wanted := range filter {
|
||||
got, exists := metadata[key]
|
||||
if !exists {
|
||||
return false
|
||||
}
|
||||
if !equalAny(got, wanted) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func equalAny(left any, right any) bool {
|
||||
return toString(left) == toString(right)
|
||||
}
|
||||
|
||||
func toString(v any) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return fmtAny(v)
|
||||
}
|
||||
|
||||
func fmtAny(v any) string {
|
||||
return strings.TrimSpace(fmt.Sprintf("%v", v))
|
||||
}
|
||||
35
backend/infra/rag/store/milvus_store.go
Normal file
35
backend/infra/rag/store/milvus_store.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/LoveLosita/smartflow/backend/infra/rag/core"
|
||||
)
|
||||
|
||||
// MilvusStore 是 Milvus 连接器占位实现。
|
||||
//
|
||||
// 说明:
|
||||
// 1. 本轮先保留接口结构,便于后续平滑替换 InMemoryStore;
|
||||
// 2. 真实接入时需补充连接池、集合初始化、元数据过滤与错误转换。
|
||||
type MilvusStore struct{}
|
||||
|
||||
func NewMilvusStore() *MilvusStore {
|
||||
return &MilvusStore{}
|
||||
}
|
||||
|
||||
func (s *MilvusStore) Upsert(_ context.Context, _ []core.VectorRow) error {
|
||||
return errors.New("milvus store is not implemented yet")
|
||||
}
|
||||
|
||||
func (s *MilvusStore) Search(_ context.Context, _ core.VectorSearchRequest) ([]core.ScoredVectorRow, error) {
|
||||
return nil, errors.New("milvus store is not implemented yet")
|
||||
}
|
||||
|
||||
func (s *MilvusStore) Delete(_ context.Context, _ []string) error {
|
||||
return errors.New("milvus store is not implemented yet")
|
||||
}
|
||||
|
||||
func (s *MilvusStore) Get(_ context.Context, _ []string) ([]core.VectorRow, error) {
|
||||
return nil, errors.New("milvus store is not implemented yet")
|
||||
}
|
||||
8
backend/infra/rag/store/vector_store.go
Normal file
8
backend/infra/rag/store/vector_store.go
Normal file
@@ -0,0 +1,8 @@
|
||||
package store
|
||||
|
||||
import "github.com/LoveLosita/smartflow/backend/infra/rag/core"
|
||||
|
||||
// EnsureCompile 用于静态校验实现是否满足接口。
|
||||
func EnsureCompile() {
|
||||
var _ core.VectorStore = (*InMemoryVectorStore)(nil)
|
||||
}
|
||||
Reference in New Issue
Block a user