214 lines
9.0 KiB
Go
214 lines
9.0 KiB
Go
package providermemory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"data-terminal-cloud/internal/audit"
|
|
"data-terminal-cloud/internal/database"
|
|
"data-terminal-cloud/internal/outbox"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
var ErrMemoryChangedDuringProviderSync = errors.New("memory changed during provider synchronization")
|
|
|
|
type Processor struct {
|
|
pool *pgxpool.Pool
|
|
client Client
|
|
audit *audit.Repository
|
|
}
|
|
|
|
type providerSnapshot struct {
|
|
tenantInternalID string
|
|
tenantID string
|
|
householdID string
|
|
subjectID string
|
|
memoryID string
|
|
memoryVersionID string
|
|
content string
|
|
contentDigest string
|
|
providerRef string
|
|
status string
|
|
currentVersionID string
|
|
sensitivity string
|
|
versionNo int64
|
|
consentEffective bool
|
|
}
|
|
|
|
func NewProcessor(pool *pgxpool.Pool, client Client) (*Processor, error) {
|
|
if pool == nil || client == nil {
|
|
return nil, errors.New("provider memory processor requires database and client")
|
|
}
|
|
return &Processor{pool: pool, client: client, audit: audit.NewRepository()}, nil
|
|
}
|
|
|
|
func (p *Processor) HandleSync(ctx context.Context, event outbox.Event) error {
|
|
var memoryVersionID string
|
|
if err := json.Unmarshal(event.Payload, &struct {
|
|
MemoryVersionID *string `json:"memory_version_id"`
|
|
}{MemoryVersionID: &memoryVersionID}); err != nil || memoryVersionID == "" {
|
|
return errors.New("provider memory sync event requires memory_version_id")
|
|
}
|
|
before, err := p.loadSnapshot(ctx, p.pool, memoryVersionID, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !providerEligible(before, memoryVersionID) {
|
|
if before.providerRef != "" {
|
|
if err := p.client.Delete(ctx, before.providerRef, "provider-memory-stale:"+memoryVersionID); err != nil {
|
|
return p.markFailed(ctx, memoryVersionID, err)
|
|
}
|
|
}
|
|
_, err := p.pool.Exec(ctx, `UPDATE memory_index_entry SET status='STALE',provider_ref=NULL,last_attempt_at=now(),last_error='memory is not currently eligible',updated_at=now() WHERE memory_version_id=$1::uuid AND index_type='provider'`, memoryVersionID)
|
|
return err
|
|
}
|
|
if err := p.deleteSupersededRefs(ctx, before.memoryID, memoryVersionID); err != nil {
|
|
return p.markFailed(ctx, memoryVersionID, err)
|
|
}
|
|
providerRef, err := p.client.Upsert(ctx, Memory{
|
|
TenantID: before.tenantID, HouseholdID: before.householdID, SubjectID: before.subjectID,
|
|
MemoryID: before.memoryID, MemoryVersion: before.memoryVersionID, VersionNo: before.versionNo,
|
|
Content: before.content, ContentDigest: before.contentDigest, IdempotencyKey: "provider-memory-upsert:" + memoryVersionID,
|
|
})
|
|
if err != nil {
|
|
return p.markFailed(ctx, memoryVersionID, err)
|
|
}
|
|
|
|
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback(context.Background())
|
|
current, err := p.loadSnapshot(ctx, tx, memoryVersionID, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !providerEligible(current, memoryVersionID) || current.contentDigest != before.contentDigest {
|
|
_ = p.client.Delete(ctx, providerRef, "provider-memory-rollback:"+memoryVersionID)
|
|
return outbox.Retryable(ErrMemoryChangedDuringProviderSync)
|
|
}
|
|
if _, err := tx.Exec(ctx, `UPDATE memory_index_entry SET status='ACTIVE',provider_ref=$2,indexed_at=now(),last_attempt_at=now(),last_error=NULL,updated_at=now() WHERE memory_version_id=$1::uuid AND index_type='provider'`, memoryVersionID, providerRef); err != nil {
|
|
return err
|
|
}
|
|
if err := p.audit.Record(ctx, tx, audit.Event{TenantID: ¤t.tenantInternalID, ActorType: "system", Action: "memory.provider_sync.complete", ResourceType: "memory", ResourceID: ¤t.memoryID, Result: "success", Detail: map[string]any{"memory_version_id": current.memoryVersionID}}); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit(ctx)
|
|
}
|
|
|
|
func (p *Processor) HandleDelete(ctx context.Context, event outbox.Event) error {
|
|
rows, err := p.pool.Query(ctx, `SELECT memory_version_id::text,COALESCE(provider_ref,'') FROM memory_index_entry WHERE memory_id=$1::uuid AND index_type='provider' AND status<>'DELETED' ORDER BY created_at,id`, event.AggregateID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
type reference struct{ versionID, providerRef string }
|
|
refs := make([]reference, 0)
|
|
for rows.Next() {
|
|
var item reference
|
|
if err := rows.Scan(&item.versionID, &item.providerRef); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
refs = append(refs, item)
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
for _, item := range refs {
|
|
if item.providerRef != "" {
|
|
if err := p.client.Delete(ctx, item.providerRef, "provider-memory-delete:"+item.versionID); err != nil {
|
|
return p.markFailed(ctx, item.versionID, err)
|
|
}
|
|
}
|
|
}
|
|
return database.Transact(ctx, p.pool, func(tx pgx.Tx) error {
|
|
var tenantID, memoryPublicID string
|
|
err := tx.QueryRow(ctx, `SELECT tenant_id::text,public_id::text FROM curated_memory WHERE id=$1::uuid FOR UPDATE`, event.AggregateID).Scan(&tenantID, &memoryPublicID)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(ctx, `UPDATE memory_index_entry SET embedding=NULL,provider_ref=NULL,status='DELETED',indexed_at=NULL,last_attempt_at=now(),last_error=NULL,updated_at=now() WHERE memory_id=$1::uuid AND index_type='provider'`, event.AggregateID); err != nil {
|
|
return err
|
|
}
|
|
return p.audit.Record(ctx, tx, audit.Event{TenantID: &tenantID, ActorType: "system", Action: "memory.provider_delete.complete", ResourceType: "memory", ResourceID: &memoryPublicID, Result: "success", Detail: map[string]any{"provider_refs_deleted": len(refs)}})
|
|
})
|
|
}
|
|
|
|
type snapshotQueryer interface {
|
|
QueryRow(context.Context, string, ...any) pgx.Row
|
|
}
|
|
|
|
func (p *Processor) loadSnapshot(ctx context.Context, q snapshotQueryer, versionID string, lock bool) (providerSnapshot, error) {
|
|
lockClause := ""
|
|
if lock {
|
|
lockClause = " FOR UPDATE OF mi,cm,sp"
|
|
}
|
|
var s providerSnapshot
|
|
err := q.QueryRow(ctx, `SELECT t.id::text,t.public_id::text,h.public_id::text,sp.public_id::text,cm.public_id::text,mv.public_id::text,
|
|
mv.content,mv.content_digest,COALESCE(mi.provider_ref,''),cm.status,COALESCE(cm.current_version_id::text,''),cm.sensitivity,mv.version_no,
|
|
EXISTS(SELECT 1 FROM consent_record cr WHERE cr.tenant_id=cm.tenant_id AND cr.subject_id=cm.subject_id AND cr.consent_type='personalized_memory' AND cr.withdrawn_at IS NULL AND (cr.expires_at IS NULL OR cr.expires_at>now()))
|
|
FROM memory_index_entry mi JOIN memory_version mv ON mv.id=mi.memory_version_id JOIN curated_memory cm ON cm.id=mi.memory_id
|
|
JOIN subject_profile sp ON sp.id=cm.subject_id AND sp.tenant_id=cm.tenant_id AND sp.status IN ('active','suspended')
|
|
JOIN household h ON h.id=sp.household_id AND h.tenant_id=sp.tenant_id JOIN tenant t ON t.id=cm.tenant_id
|
|
WHERE mi.memory_version_id=$1::uuid AND mi.index_type='provider'`+lockClause, versionID).Scan(
|
|
&s.tenantInternalID, &s.tenantID, &s.householdID, &s.subjectID, &s.memoryID, &s.memoryVersionID, &s.content, &s.contentDigest, &s.providerRef, &s.status, &s.currentVersionID, &s.sensitivity, &s.versionNo, &s.consentEffective)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return providerSnapshot{}, errors.New("provider memory index entry not found")
|
|
}
|
|
return s, err
|
|
}
|
|
|
|
func providerEligible(s providerSnapshot, internalVersionID string) bool {
|
|
return s.status == "ACTIVE" && s.currentVersionID == internalVersionID && s.consentEffective && s.sensitivity == "normal"
|
|
}
|
|
|
|
func (p *Processor) deleteSupersededRefs(ctx context.Context, memoryPublicID, currentVersionID string) error {
|
|
rows, err := p.pool.Query(ctx, `SELECT memory_version_id::text,provider_ref FROM memory_index_entry mi JOIN curated_memory cm ON cm.id=mi.memory_id WHERE cm.public_id=$1::uuid AND mi.index_type='provider' AND mi.memory_version_id<>$2::uuid AND mi.provider_ref IS NOT NULL AND mi.status<>'DELETED'`, memoryPublicID, currentVersionID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
type ref struct{ versionID, providerRef string }
|
|
items := []ref{}
|
|
for rows.Next() {
|
|
var item ref
|
|
if err := rows.Scan(&item.versionID, &item.providerRef); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
items = append(items, item)
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
for _, item := range items {
|
|
if err := p.client.Delete(ctx, item.providerRef, "provider-memory-supersede:"+item.versionID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if len(items) > 0 {
|
|
_, err = p.pool.Exec(ctx, `UPDATE memory_index_entry mi SET status='DELETED',provider_ref=NULL,last_error=NULL,updated_at=now() FROM curated_memory cm WHERE mi.memory_id=cm.id AND cm.public_id=$1::uuid AND mi.index_type='provider' AND mi.memory_version_id<>$2::uuid`, memoryPublicID, currentVersionID)
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (p *Processor) markFailed(ctx context.Context, versionID string, cause error) error {
|
|
message := cause.Error()
|
|
if len(message) > 450 {
|
|
message = message[:450]
|
|
}
|
|
_, _ = p.pool.Exec(ctx, `UPDATE memory_index_entry SET status='FAILED',last_attempt_at=now(),last_error=$2,updated_at=now() WHERE memory_version_id=$1::uuid AND index_type='provider'`, versionID, message)
|
|
if outbox.IsRetryable(cause) {
|
|
return cause
|
|
}
|
|
return fmt.Errorf("provider memory synchronization failed: %w", cause)
|
|
}
|