chore: establish local platform baseline
This commit is contained in:
Codex Local
@@ -0,0 +1,120 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type AccessProcessor struct {
|
||||
pool *pgxpool.Pool
|
||||
audit *audit.Repository
|
||||
}
|
||||
|
||||
func NewAccessProcessor(pool *pgxpool.Pool) (*AccessProcessor, error) {
|
||||
if pool == nil {
|
||||
return nil, errors.New("database pool is required")
|
||||
}
|
||||
return &AccessProcessor{pool: pool, audit: audit.NewRepository()}, nil
|
||||
}
|
||||
|
||||
func (p *AccessProcessor) Handle(ctx context.Context, event outbox.Event) error {
|
||||
return database.Transact(ctx, p.pool, func(tx pgx.Tx) error {
|
||||
var tenantID, subjectID, publicID, status, requestType string
|
||||
var scopeJSON []byte
|
||||
if err := tx.QueryRow(ctx, `SELECT tenant_id::text,subject_id::text,public_id::text,status,request_type,scope FROM privacy_request WHERE id=$1::uuid FOR UPDATE`, event.AggregateID).Scan(&tenantID, &subjectID, &publicID, &status, &requestType, &scopeJSON); errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("load privacy access request: %w", err)
|
||||
}
|
||||
if status == "COMPLETED" {
|
||||
return nil
|
||||
}
|
||||
if requestType != "ACCESS" {
|
||||
return fmt.Errorf("privacy request type %s is not handled by access processor", requestType)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE privacy_request SET status='VERIFYING',version=version+1,updated_at=now() WHERE id=$1::uuid AND status='SUBMITTED'`, event.AggregateID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE privacy_request SET status='EXECUTING',version=version+1,updated_at=now() WHERE id=$1::uuid AND status IN ('VERIFYING','PARTIAL_FAILED')`, event.AggregateID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE privacy_subtask SET status='RUNNING',attempts=attempts+1,last_error=NULL,updated_at=now() WHERE privacy_request_id=$1::uuid AND subsystem='postgresql' AND target_type='subject_data_inventory'`, event.AggregateID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
counts, err := collectInventory(ctx, tx, tenantID, subjectID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var scope map[string]any
|
||||
if err := json.Unmarshal(scopeJSON, &scope); err != nil {
|
||||
return fmt.Errorf("decode privacy scope: %w", err)
|
||||
}
|
||||
result := map[string]any{
|
||||
"inventory_counts": filterInventory(counts, scope),
|
||||
"generated_at": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"content_included": false,
|
||||
}
|
||||
resultJSON, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE privacy_subtask SET status='COMPLETED',completed_at=now(),updated_at=now() WHERE privacy_request_id=$1::uuid AND subsystem='postgresql' AND target_type='subject_data_inventory'`, event.AggregateID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE privacy_request SET status='COMPLETED',result_summary=$2::jsonb,completed_at=now(),version=version+1,updated_at=now() WHERE id=$1::uuid`, event.AggregateID, resultJSON); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.audit.Record(ctx, tx, audit.Event{TenantID: &tenantID, ActorType: "system", Action: "privacy.access.complete", ResourceType: "privacy_request", ResourceID: &publicID, Result: "success", Detail: map[string]any{"subject_digest_scope": "tenant_subject", "content_included": false}})
|
||||
})
|
||||
}
|
||||
|
||||
func collectInventory(ctx context.Context, tx pgx.Tx, tenantID, subjectID string) (map[string]int64, error) {
|
||||
queries := map[string]string{
|
||||
"profile": `SELECT count(*) FROM subject_profile WHERE tenant_id=$1::uuid AND id=$2::uuid`,
|
||||
"consents": `SELECT count(*) FROM consent_record WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
"device_bindings": `SELECT count(*) FROM device_binding WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
"conversations": `SELECT count(*) FROM conversation WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
"transcripts": `SELECT count(*) FROM conversation_turn WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
"summaries": `SELECT count(*) FROM conversation_summary WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
"stories": `SELECT count(*) FROM story_draft WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
"memories": `SELECT count(*) FROM curated_memory WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
}
|
||||
result := make(map[string]int64, len(queries))
|
||||
for category, query := range queries {
|
||||
var count int64
|
||||
if err := tx.QueryRow(ctx, query, tenantID, subjectID).Scan(&count); err != nil {
|
||||
return nil, fmt.Errorf("count privacy inventory category %s: %w", category, err)
|
||||
}
|
||||
result[category] = count
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func filterInventory(counts map[string]int64, scope map[string]any) map[string]int64 {
|
||||
requested, ok := scope["categories"].([]any)
|
||||
if !ok || len(requested) == 0 {
|
||||
return counts
|
||||
}
|
||||
result := make(map[string]int64, len(requested))
|
||||
for _, value := range requested {
|
||||
category, ok := value.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if count, exists := counts[category]; exists {
|
||||
result[category] = count
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const (
|
||||
privacyAccessRequestID = "90000000-0000-0000-0000-000000000701"
|
||||
privacyAccessRequestPublicID = "90000000-0000-0000-0000-000000000711"
|
||||
privacyWrongRequestID = "90000000-0000-0000-0000-000000000702"
|
||||
)
|
||||
|
||||
func TestAccessProcessorCompletesScopedInventoryIdempotently(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "privacy-access-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
cleanupPrivacyAccessFixture(t, ctx, pool)
|
||||
defer cleanupPrivacyAccessFixture(t, context.Background(), pool)
|
||||
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO privacy_request (
|
||||
id,public_id,tenant_id,subject_id,request_type,scope,requested_by,idempotency_key,request_digest
|
||||
) VALUES (
|
||||
$1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000401',
|
||||
'ACCESS','{"categories":["profile","conversations","unknown",7]}'::jsonb,
|
||||
'10000000-0000-0000-0000-000000000101','privacy-access-integration',repeat('7',64)
|
||||
)`, privacyAccessRequestID, privacyAccessRequestPublicID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO privacy_subtask (
|
||||
public_id,privacy_request_id,subsystem,target_type,target_ref
|
||||
) VALUES (
|
||||
'90000000-0000-0000-0000-000000000721',$1::uuid,'postgresql','subject_data_inventory','subject'
|
||||
)`, privacyAccessRequestID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
processor, err := NewAccessProcessor(pool)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := processor.Handle(ctx, outbox.Event{AggregateID: privacyAccessRequestID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var status, taskStatus string
|
||||
var attempts int
|
||||
var resultBody []byte
|
||||
if err := pool.QueryRow(ctx, `SELECT pr.status,pr.result_summary,ps.status,ps.attempts
|
||||
FROM privacy_request pr JOIN privacy_subtask ps ON ps.privacy_request_id=pr.id WHERE pr.id=$1::uuid`, privacyAccessRequestID).Scan(&status, &resultBody, &taskStatus, &attempts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var result struct {
|
||||
InventoryCounts map[string]int64 `json:"inventory_counts"`
|
||||
ContentIncluded bool `json:"content_included"`
|
||||
}
|
||||
if err := json.Unmarshal(resultBody, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "COMPLETED" || taskStatus != "COMPLETED" || attempts != 1 || result.ContentIncluded || len(result.InventoryCounts) != 2 || result.InventoryCounts["profile"] != 1 || result.InventoryCounts["conversations"] != 0 {
|
||||
t.Fatalf("status=%s task=%s attempts=%d result=%#v", status, taskStatus, attempts, result)
|
||||
}
|
||||
if err := processor.Handle(ctx, outbox.Event{AggregateID: privacyAccessRequestID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var audits int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM audit_event WHERE action='privacy.access.complete' AND resource_id=$1::uuid`, privacyAccessRequestPublicID).Scan(&audits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if audits != 1 {
|
||||
t.Fatalf("privacy access audits=%d", audits)
|
||||
}
|
||||
if err := processor.Handle(ctx, outbox.Event{AggregateID: "90000000-0000-0000-0000-000000000799"}); err != nil {
|
||||
t.Fatalf("missing request error=%v", err)
|
||||
}
|
||||
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO privacy_request (
|
||||
id,public_id,tenant_id,subject_id,request_type,scope,requested_by,idempotency_key,request_digest
|
||||
) VALUES (
|
||||
$1::uuid,'90000000-0000-0000-0000-000000000712','10000000-0000-0000-0000-000000000001',
|
||||
'10000000-0000-0000-0000-000000000401','CORRECTION','{}'::jsonb,
|
||||
'10000000-0000-0000-0000-000000000101','privacy-access-wrong-type',repeat('8',64)
|
||||
)`, privacyWrongRequestID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := processor.Handle(ctx, outbox.Event{AggregateID: privacyWrongRequestID}); err == nil || !strings.Contains(err.Error(), "CORRECTION") {
|
||||
t.Fatalf("wrong request type error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupPrivacyAccessFixture(t *testing.T, ctx context.Context, pool *pgxpool.Pool) {
|
||||
t.Helper()
|
||||
statements := []string{
|
||||
`DELETE FROM audit_event WHERE action='privacy.access.complete' AND resource_id IN ('90000000-0000-0000-0000-000000000711'::uuid,'90000000-0000-0000-0000-000000000712'::uuid)`,
|
||||
`DELETE FROM privacy_request WHERE id IN ('90000000-0000-0000-0000-000000000701'::uuid,'90000000-0000-0000-0000-000000000702'::uuid)`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := pool.Exec(ctx, statement); err != nil {
|
||||
t.Fatalf("cleanup privacy access fixture: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package privacy
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFilterInventoryUsesRequestedCategories(t *testing.T) {
|
||||
counts := map[string]int64{"profile": 1, "conversations": 3, "memories": 2}
|
||||
filtered := filterInventory(counts, map[string]any{"categories": []any{"profile", "memories", "unknown", 12}})
|
||||
if len(filtered) != 2 || filtered["profile"] != 1 || filtered["memories"] != 2 {
|
||||
t.Fatalf("filtered inventory = %#v", filtered)
|
||||
}
|
||||
if _, exists := filtered["conversations"]; exists {
|
||||
t.Fatal("unrequested category must not be returned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterInventoryDefaultsToAllCategories(t *testing.T) {
|
||||
counts := map[string]int64{"profile": 1, "conversations": 3}
|
||||
filtered := filterInventory(counts, map[string]any{})
|
||||
if len(filtered) != len(counts) || filtered["conversations"] != 3 {
|
||||
t.Fatalf("default inventory = %#v", filtered)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type deletionObjectStore interface {
|
||||
Remove(context.Context, string) error
|
||||
}
|
||||
|
||||
type DeleteProcessor struct {
|
||||
pool *pgxpool.Pool
|
||||
redis *redis.Client
|
||||
store deletionObjectStore
|
||||
audit *audit.Repository
|
||||
}
|
||||
|
||||
type deletionRequest struct {
|
||||
id, tenantID, subjectID, subjectPublicID, publicID, status, requestType string
|
||||
}
|
||||
|
||||
type deletionSubtask struct {
|
||||
id, subsystem, status string
|
||||
}
|
||||
|
||||
func NewDeleteProcessor(pool *pgxpool.Pool, redisClient *redis.Client, store deletionObjectStore) (*DeleteProcessor, error) {
|
||||
if pool == nil || redisClient == nil || store == nil {
|
||||
return nil, errors.New("database, redis and object store are required")
|
||||
}
|
||||
return &DeleteProcessor{pool: pool, redis: redisClient, store: store, audit: audit.NewRepository()}, nil
|
||||
}
|
||||
|
||||
func (p *DeleteProcessor) Handle(ctx context.Context, event outbox.Event) error {
|
||||
request, err := p.loadRequest(ctx, event.AggregateID)
|
||||
if err != nil || request.id == "" || request.status == "COMPLETED" {
|
||||
return err
|
||||
}
|
||||
if request.requestType != "DELETE" {
|
||||
return fmt.Errorf("privacy request type %s is not handled by delete processor", request.requestType)
|
||||
}
|
||||
if _, err := p.pool.Exec(ctx, `UPDATE privacy_request SET status='VERIFYING',version=version+1,updated_at=now() WHERE id=$1::uuid AND status='SUBMITTED'`, request.id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := p.pool.Exec(ctx, `UPDATE privacy_request SET status='EXECUTING',completed_at=NULL,version=version+1,updated_at=now() WHERE id=$1::uuid AND status IN ('VERIFYING','PARTIAL_FAILED')`, request.id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tasks, err := p.loadSubtasks(ctx, request.id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, task := range tasks {
|
||||
if task.status == "COMPLETED" || task.status == "SKIPPED" {
|
||||
continue
|
||||
}
|
||||
if _, err := p.pool.Exec(ctx, `UPDATE privacy_subtask SET status='RUNNING',attempts=attempts+1,next_attempt_at=NULL,last_error=NULL,completed_at=NULL,updated_at=now() WHERE id=$1::uuid`, task.id); err != nil {
|
||||
return err
|
||||
}
|
||||
outcome, note, runErr := p.executeSubtask(ctx, request, task.subsystem)
|
||||
if runErr != nil {
|
||||
message := safeDeletionError(runErr)
|
||||
_, _ = p.pool.Exec(ctx, `UPDATE privacy_subtask SET status='FAILED',last_error=$2,next_attempt_at=now()+interval '30 seconds',updated_at=now() WHERE id=$1::uuid`, task.id, message)
|
||||
_, _ = p.pool.Exec(ctx, `UPDATE privacy_request SET status='PARTIAL_FAILED',result_summary=jsonb_build_object('failed_subsystem',$2,'content_included',false),version=version+1,updated_at=now() WHERE id=$1::uuid`, request.id, task.subsystem)
|
||||
return fmt.Errorf("delete subject subsystem %s: %w", task.subsystem, runErr)
|
||||
}
|
||||
if _, err := p.pool.Exec(ctx, `UPDATE privacy_subtask SET status=$2,last_error=NULLIF($3,''),next_attempt_at=NULL,completed_at=now(),updated_at=now() WHERE id=$1::uuid`, task.id, outcome, note); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return database.Transact(ctx, p.pool, func(tx pgx.Tx) error {
|
||||
var open int
|
||||
if err := tx.QueryRow(ctx, `SELECT count(*) FROM privacy_subtask WHERE privacy_request_id=$1::uuid AND status NOT IN ('COMPLETED','SKIPPED')`, request.id).Scan(&open); err != nil {
|
||||
return err
|
||||
}
|
||||
if open != 0 {
|
||||
return errors.New("privacy deletion reconciliation still has open subtasks")
|
||||
}
|
||||
result, _ := json.Marshal(map[string]any{"subtasks_reconciled": len(tasks), "content_included": false, "completed_at": time.Now().UTC()})
|
||||
if _, err := tx.Exec(ctx, `UPDATE privacy_request SET status='COMPLETED',result_summary=$2::jsonb,completed_at=now(),version=version+1,updated_at=now() WHERE id=$1::uuid`, request.id, result); err != nil {
|
||||
return err
|
||||
}
|
||||
digest := sha256.Sum256([]byte(request.subjectPublicID))
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO tombstone(tenant_id,resource_type,resource_digest,deletion_request_id,retention_reason) VALUES($1::uuid,'subject',$2,$3::uuid,'privacy deletion proof; contains no subject content') ON CONFLICT (tenant_id,resource_type,resource_digest) DO NOTHING`, request.tenantID, hex.EncodeToString(digest[:]), request.id); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.audit.Record(ctx, tx, audit.Event{TenantID: &request.tenantID, ActorType: "system", Action: "privacy.delete.complete", ResourceType: "privacy_request", ResourceID: &request.publicID, Result: "success", Detail: map[string]any{"subject_digest": hex.EncodeToString(digest[:]), "subtasks_reconciled": len(tasks), "content_included": false}})
|
||||
})
|
||||
}
|
||||
|
||||
func (p *DeleteProcessor) loadRequest(ctx context.Context, id string) (deletionRequest, error) {
|
||||
var value deletionRequest
|
||||
err := p.pool.QueryRow(ctx, `SELECT pr.id::text,pr.tenant_id::text,pr.subject_id::text,sp.public_id::text,pr.public_id::text,pr.status,pr.request_type FROM privacy_request pr JOIN subject_profile sp ON sp.id=pr.subject_id WHERE pr.id=$1::uuid`, id).Scan(&value.id, &value.tenantID, &value.subjectID, &value.subjectPublicID, &value.publicID, &value.status, &value.requestType)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return deletionRequest{}, nil
|
||||
}
|
||||
return value, err
|
||||
}
|
||||
|
||||
func (p *DeleteProcessor) loadSubtasks(ctx context.Context, requestID string) ([]deletionSubtask, error) {
|
||||
rows, err := p.pool.Query(ctx, `SELECT id::text,subsystem,status FROM privacy_subtask WHERE privacy_request_id=$1::uuid ORDER BY CASE subsystem WHEN 'redis' THEN 1 WHEN 'export_files' THEN 2 WHEN 'object_storage' THEN 3 WHEN 'pgvector' THEN 4 WHEN 'provider_index' THEN 5 WHEN 'postgresql' THEN 6 ELSE 99 END,id`, requestID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []deletionSubtask{}
|
||||
for rows.Next() {
|
||||
var item deletionSubtask
|
||||
if err := rows.Scan(&item.id, &item.subsystem, &item.status); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (p *DeleteProcessor) executeSubtask(ctx context.Context, request deletionRequest, subsystem string) (string, string, error) {
|
||||
switch subsystem {
|
||||
case "redis":
|
||||
keys, err := p.subjectRedisKeys(ctx, request.tenantID, request.subjectID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if len(keys) > 0 {
|
||||
if err := p.redis.Del(ctx, keys...).Err(); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
return "COMPLETED", "", nil
|
||||
case "export_files":
|
||||
rows, err := p.pool.Query(ctx, `SELECT object_ref FROM export_task WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND object_ref IS NOT NULL`, request.tenantID, request.subjectID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
refs := []string{}
|
||||
for rows.Next() {
|
||||
var ref string
|
||||
if err := rows.Scan(&ref); err != nil {
|
||||
rows.Close()
|
||||
return "", "", err
|
||||
}
|
||||
refs = append(refs, ref)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
for _, ref := range refs {
|
||||
if err := p.store.Remove(ctx, ref); err != nil {
|
||||
return "", "", outbox.Retryable(err)
|
||||
}
|
||||
}
|
||||
_, err = p.pool.Exec(ctx, `UPDATE export_task SET status='EXPIRED',scope='{}'::jsonb,object_ref=NULL,object_size=NULL,content_digest=NULL,expires_at=NULL,last_error=NULL,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`, request.tenantID, request.subjectID)
|
||||
return "COMPLETED", "", err
|
||||
case "object_storage":
|
||||
rows, err := p.pool.Query(ctx, `SELECT ms.object_ref FROM media_segment ms JOIN conversation c ON c.id=ms.conversation_id WHERE c.tenant_id=$1::uuid AND c.subject_id=$2::uuid AND ms.object_ref IS NOT NULL`, request.tenantID, request.subjectID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
refs := []string{}
|
||||
for rows.Next() {
|
||||
var ref string
|
||||
if err := rows.Scan(&ref); err != nil {
|
||||
rows.Close()
|
||||
return "", "", err
|
||||
}
|
||||
refs = append(refs, ref)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
for _, ref := range refs {
|
||||
if err := p.store.Remove(ctx, ref); err != nil {
|
||||
return "", "", outbox.Retryable(err)
|
||||
}
|
||||
}
|
||||
_, err = p.pool.Exec(ctx, `UPDATE media_segment ms SET status='DELETED',object_ref=NULL,expected_digest=repeat('0',64),expected_size=1,duration_ms=1,last_error_code=NULL,deleted_at=COALESCE(deleted_at,now()),updated_at=now() FROM conversation c WHERE ms.conversation_id=c.id AND c.tenant_id=$1::uuid AND c.subject_id=$2::uuid`, request.tenantID, request.subjectID)
|
||||
return "COMPLETED", "", err
|
||||
case "pgvector":
|
||||
_, err := p.pool.Exec(ctx, `UPDATE memory_index_entry SET embedding=NULL,provider_ref=NULL,status='DELETED',last_error=NULL,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND index_type='pgvector'`, request.tenantID, request.subjectID)
|
||||
return "COMPLETED", "", err
|
||||
case "provider_index":
|
||||
var unresolved int
|
||||
if err := p.pool.QueryRow(ctx, `SELECT count(*) FROM memory_index_entry WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND index_type='provider' AND provider_ref IS NOT NULL AND status<>'DELETED'`, request.tenantID, request.subjectID).Scan(&unresolved); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if unresolved > 0 {
|
||||
return "", "", errors.New("external provider index references exist but no deletion adapter is configured")
|
||||
}
|
||||
_, err := p.pool.Exec(ctx, `UPDATE memory_index_entry SET embedding=NULL,status='DELETED',last_error=NULL,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND index_type='provider'`, request.tenantID, request.subjectID)
|
||||
return "SKIPPED", "no external provider index references were present", err
|
||||
case "postgresql":
|
||||
return "COMPLETED", "", p.redactPostgreSQL(ctx, request)
|
||||
default:
|
||||
return "", "", fmt.Errorf("unsupported deletion subsystem %q", subsystem)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *DeleteProcessor) subjectRedisKeys(ctx context.Context, tenantID, subjectID string) ([]string, error) {
|
||||
rows, err := p.pool.Query(ctx, `SELECT DISTINCT c.public_id::text,d.public_id::text FROM conversation c JOIN device d ON d.id=c.device_id AND d.tenant_id=c.tenant_id WHERE c.tenant_id=$1::uuid AND c.subject_id=$2::uuid`, tenantID, subjectID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
keys := []string{}
|
||||
for rows.Next() {
|
||||
var conversationID, deviceID string
|
||||
if err := rows.Scan(&conversationID, &deviceID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys = append(keys, "voice-platform:conversation:lease:"+conversationID, "voice-platform:device:presence:"+deviceID)
|
||||
}
|
||||
return keys, rows.Err()
|
||||
}
|
||||
|
||||
func (p *DeleteProcessor) redactPostgreSQL(ctx context.Context, request deletionRequest) error {
|
||||
return database.Transact(ctx, p.pool, func(tx pgx.Tx) error {
|
||||
statements := []string{
|
||||
`UPDATE subject_fact SET content='[已按隐私请求删除]',source_ref=NULL,confirmation_status='superseded',valid_to=COALESCE(valid_to,now()),version=version+1,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE conversation_turn SET final_text='[已按隐私请求删除]',provider_ref=NULL WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE conversation_summary SET content='[已按隐私请求删除]',source_turn_ids='["privacy-redacted"]'::jsonb,status='SUPERSEDED',last_error=NULL WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE story_revision sr SET content='[已按隐私请求删除]',edit_reason=NULL,model_ref=NULL,prompt_version=NULL FROM story_draft sd WHERE sr.story_id=sd.id AND sd.tenant_id=$1::uuid AND sd.subject_id=$2::uuid`,
|
||||
`UPDATE story_draft SET title='[已按隐私请求删除]',status='SUPERSEDED',review_reason=NULL,version=version+1,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE fact_candidate SET content='[已按隐私请求删除]',time_range='{}'::jsonb,source_turn_ids='["privacy-redacted"]'::jsonb,status='SUPERSEDED',review_reason=NULL,version=version+1,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE memory_version mv SET content='[已按隐私请求删除]',source_refs='[{"type":"privacy_tombstone"}]'::jsonb,change_reason=NULL,content_digest=encode(digest('[已按隐私请求删除]','sha256'),'hex') FROM curated_memory cm WHERE mv.memory_id=cm.id AND cm.tenant_id=$1::uuid AND cm.subject_id=$2::uuid`,
|
||||
`UPDATE curated_memory SET status='DELETED',valid_to=COALESCE(valid_to,now()),deleted_at=COALESCE(deleted_at,now()),version=version+1,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE memory_index_entry SET embedding=NULL,provider_ref=NULL,status='DELETED',last_error=NULL,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE consent_snapshot SET effective_consents='{}'::jsonb,source_consent_ids='[]'::jsonb WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE context_snapshot SET memory_versions='[]'::jsonb,content_digest=repeat('0',64),sanitized_summary=NULL WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE recording_access_grant SET purpose='[privacy-redacted]',revoked_at=COALESCE(revoked_at,now()),updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE sharing_grant SET purpose='[privacy-redacted]',revoked_at=COALESCE(revoked_at,now()),version=version+1,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE sharing_access_log sal SET reason_code=CASE WHEN result='DENIED' THEN 'PRIVACY_REDACTED' ELSE NULL END FROM sharing_grant sg WHERE sal.sharing_grant_id=sg.id AND sg.tenant_id=$1::uuid AND sg.subject_id=$2::uuid`,
|
||||
`UPDATE media_access_log mal SET purpose='[privacy-redacted]',reason_code=NULL FROM conversation c WHERE mal.conversation_id=c.id AND c.tenant_id=$1::uuid AND c.subject_id=$2::uuid`,
|
||||
`UPDATE upload_batch ub SET manifest_digest=repeat('0',64),last_error_code=NULL,status=CASE WHEN ub.status='COMPLETED' THEN ub.status ELSE 'EXPIRED' END,updated_at=now() FROM conversation c WHERE ub.conversation_id=c.id AND c.tenant_id=$1::uuid AND c.subject_id=$2::uuid`,
|
||||
`UPDATE provider_event pe SET sanitized_payload='{}'::jsonb,last_error=NULL FROM conversation c WHERE pe.conversation_id=c.id AND c.tenant_id=$1::uuid AND c.subject_id=$2::uuid`,
|
||||
`UPDATE device_binding SET unbound_at=COALESCE(unbound_at,now()),unbound_by=COALESCE(unbound_by,(SELECT requested_by FROM privacy_request WHERE id=$3::uuid)),reason='subject privacy deletion' WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
`UPDATE subject_profile SET preferred_name='[已删除]',profile_data='{}'::jsonb,conversation_preferences='{}'::jsonb,status='deleted',deleted_at=COALESCE(deleted_at,now()),profile_version=profile_version+1,updated_at=now() WHERE tenant_id=$1::uuid AND id=$2::uuid`,
|
||||
`DELETE FROM web_event WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
args := []any{request.tenantID, request.subjectID}
|
||||
if strings.Contains(statement, "$3") {
|
||||
args = append(args, request.id)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, statement, args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE outbox_event SET payload='{"privacy_redacted":true}'::jsonb,status=CASE WHEN status IN ('pending','processing') THEN 'completed' ELSE status END,completed_at=CASE WHEN status IN ('pending','processing') THEN COALESCE(completed_at,now()) ELSE completed_at END,locked_at=NULL,locked_by=NULL,updated_at=now() WHERE tenant_id=$1::uuid AND aggregate_id IN (SELECT id FROM conversation WHERE subject_id=$2::uuid UNION SELECT id FROM curated_memory WHERE subject_id=$2::uuid UNION SELECT id FROM export_task WHERE subject_id=$2::uuid UNION SELECT ub.public_id FROM upload_batch ub JOIN conversation c ON c.id=ub.conversation_id WHERE c.subject_id=$2::uuid)`, request.tenantID, request.subjectID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE audit_event SET purpose=NULL,detail='{"privacy_redacted":true}'::jsonb WHERE tenant_id=$1::uuid AND (detail->>'subject_id'=(SELECT public_id::text FROM subject_profile WHERE id=$2::uuid) OR resource_id IN (SELECT public_id FROM subject_profile WHERE id=$2::uuid UNION SELECT public_id FROM subject_fact WHERE subject_id=$2::uuid UNION SELECT public_id FROM consent_record WHERE subject_id=$2::uuid UNION SELECT public_id FROM conversation WHERE subject_id=$2::uuid UNION SELECT public_id FROM story_draft WHERE subject_id=$2::uuid UNION SELECT public_id FROM curated_memory WHERE subject_id=$2::uuid UNION SELECT public_id FROM export_task WHERE subject_id=$2::uuid UNION SELECT public_id FROM privacy_request WHERE subject_id=$2::uuid UNION SELECT public_id FROM recording_access_grant WHERE subject_id=$2::uuid UNION SELECT public_id FROM sharing_grant WHERE subject_id=$2::uuid UNION SELECT ub.public_id FROM upload_batch ub JOIN conversation c ON c.id=ub.conversation_id WHERE c.subject_id=$2::uuid UNION SELECT ms.public_id FROM media_segment ms JOIN conversation c ON c.id=ms.conversation_id WHERE c.subject_id=$2::uuid))`, request.tenantID, request.subjectID); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func safeDeletionError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
value := err.Error()
|
||||
if len(value) > 450 {
|
||||
return value[:450]
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"data-terminal-cloud/internal/cache"
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const (
|
||||
privacyDeleteSubjectID = "90000000-0000-0000-0000-000000000741"
|
||||
privacyDeleteSubjectPublicID = "90000000-0000-0000-0000-000000000742"
|
||||
privacyDeleteRequestID = "90000000-0000-0000-0000-000000000743"
|
||||
privacyDeleteRequestPublicID = "90000000-0000-0000-0000-000000000744"
|
||||
)
|
||||
|
||||
type privacyDeleteStore struct {
|
||||
removed []string
|
||||
}
|
||||
|
||||
func (s *privacyDeleteStore) Remove(_ context.Context, reference string) error {
|
||||
s.removed = append(s.removed, reference)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestDeleteProcessorReconcilesAllSubsystemsAndCreatesTombstone(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
redisAddress := os.Getenv("TEST_REDIS_ADDR")
|
||||
if databaseURL == "" || redisAddress == "" {
|
||||
t.Skip("TEST_DATABASE_URL and TEST_REDIS_ADDR are required")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "privacy-delete-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
cleanupPrivacyDeleteFixture(t, ctx, pool)
|
||||
defer cleanupPrivacyDeleteFixture(t, context.Background(), pool)
|
||||
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO subject_profile (
|
||||
id,public_id,tenant_id,household_id,preferred_name,status
|
||||
) VALUES (
|
||||
$1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001',
|
||||
'10000000-0000-0000-0000-000000000201','隐私删除集成长者','active'
|
||||
)`, privacyDeleteSubjectID, privacyDeleteSubjectPublicID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO privacy_request (
|
||||
id,public_id,tenant_id,subject_id,request_type,scope,requested_by,idempotency_key,request_digest
|
||||
) VALUES (
|
||||
$1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001',$3::uuid,
|
||||
'DELETE','{"reason_code":"subject_request"}'::jsonb,
|
||||
'10000000-0000-0000-0000-000000000101','privacy-delete-integration',repeat('9',64)
|
||||
)`, privacyDeleteRequestID, privacyDeleteRequestPublicID, privacyDeleteSubjectID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO export_task (
|
||||
id,public_id,tenant_id,subject_id,requested_by,scope,status,object_ref,object_size,content_digest,
|
||||
expires_at,completed_at,idempotency_key,request_digest
|
||||
) VALUES (
|
||||
'90000000-0000-0000-0000-000000000745','90000000-0000-0000-0000-000000000746',
|
||||
'10000000-0000-0000-0000-000000000001',$1::uuid,'10000000-0000-0000-0000-000000000101',
|
||||
'{}'::jsonb,'READY','privacy-delete/export.zip',128,repeat('a',64),now()+interval '1 hour',now(),
|
||||
'privacy-delete-export-integration',repeat('b',64)
|
||||
)`, privacyDeleteSubjectID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO privacy_subtask (
|
||||
public_id,privacy_request_id,subsystem,target_type,target_ref
|
||||
) SELECT gen_random_uuid(),$1::uuid,subsystem,'subject_data',subsystem
|
||||
FROM unnest(ARRAY['redis','export_files','object_storage','pgvector','provider_index','postgresql']) AS subsystem`, privacyDeleteRequestID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
redisClient := cache.Open(redisAddress, "", 0)
|
||||
defer redisClient.Close()
|
||||
store := &privacyDeleteStore{}
|
||||
processor, err := NewDeleteProcessor(pool, redisClient, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := processor.Handle(ctx, outbox.Event{AggregateID: privacyDeleteRequestID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var requestStatus, subjectStatus, preferredName string
|
||||
var completed, skipped, tombstones, audits int
|
||||
if err := pool.QueryRow(ctx, `SELECT status FROM privacy_request WHERE id=$1::uuid`, privacyDeleteRequestID).Scan(&requestStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT status,preferred_name FROM subject_profile WHERE id=$1::uuid`, privacyDeleteSubjectID).Scan(&subjectStatus, &preferredName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FILTER(WHERE status='COMPLETED'),count(*) FILTER(WHERE status='SKIPPED') FROM privacy_subtask WHERE privacy_request_id=$1::uuid`, privacyDeleteRequestID).Scan(&completed, &skipped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM tombstone WHERE deletion_request_id=$1::uuid`, privacyDeleteRequestID).Scan(&tombstones); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM audit_event WHERE action='privacy.delete.complete' AND resource_id=$1::uuid`, privacyDeleteRequestPublicID).Scan(&audits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var exportStatus string
|
||||
var exportReference *string
|
||||
if err := pool.QueryRow(ctx, `SELECT status,object_ref FROM export_task WHERE id='90000000-0000-0000-0000-000000000745'::uuid`).Scan(&exportStatus, &exportReference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if requestStatus != "COMPLETED" || subjectStatus != "deleted" || preferredName != "[已删除]" || completed != 5 || skipped != 1 || tombstones != 1 || audits != 1 || len(store.removed) != 1 || store.removed[0] != "privacy-delete/export.zip" || exportStatus != "EXPIRED" || exportReference != nil {
|
||||
t.Fatalf("request=%s subject=%s name=%q completed=%d skipped=%d tombstones=%d audits=%d removed=%v", requestStatus, subjectStatus, preferredName, completed, skipped, tombstones, audits, store.removed)
|
||||
}
|
||||
if err := processor.Handle(ctx, outbox.Event{AggregateID: privacyDeleteRequestID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := processor.Handle(ctx, outbox.Event{AggregateID: "90000000-0000-0000-0000-000000000799"}); err != nil {
|
||||
t.Fatalf("missing deletion request error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupPrivacyDeleteFixture(t *testing.T, ctx context.Context, pool *pgxpool.Pool) {
|
||||
t.Helper()
|
||||
statements := []string{
|
||||
`DELETE FROM tombstone WHERE deletion_request_id='90000000-0000-0000-0000-000000000743'::uuid`,
|
||||
`DELETE FROM audit_event WHERE action='privacy.delete.complete' AND resource_id='90000000-0000-0000-0000-000000000744'::uuid`,
|
||||
`DELETE FROM privacy_request WHERE id='90000000-0000-0000-0000-000000000743'::uuid`,
|
||||
`DELETE FROM export_task WHERE id='90000000-0000-0000-0000-000000000745'::uuid`,
|
||||
`DELETE FROM subject_profile WHERE id='90000000-0000-0000-0000-000000000741'::uuid`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := pool.Exec(ctx, statement); err != nil {
|
||||
t.Fatalf("cleanup privacy delete fixture: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDeleteProcessorValidationAndSafeErrors(t *testing.T) {
|
||||
if _, err := NewAccessProcessor(nil); err == nil {
|
||||
t.Fatal("nil access database was accepted")
|
||||
}
|
||||
if _, err := NewDeleteProcessor(nil, nil, nil); err == nil {
|
||||
t.Fatal("invalid deletion dependencies were accepted")
|
||||
}
|
||||
processor := &DeleteProcessor{}
|
||||
if _, _, err := processor.executeSubtask(context.Background(), deletionRequest{}, "unknown"); err == nil || !strings.Contains(err.Error(), "unsupported") {
|
||||
t.Fatalf("unsupported subsystem error=%v", err)
|
||||
}
|
||||
if got := safeDeletionError(nil); got != "" {
|
||||
t.Fatalf("nil error text=%q", got)
|
||||
}
|
||||
if got := safeDeletionError(errors.New("temporary object-store failure")); got != "temporary object-store failure" {
|
||||
t.Fatalf("short safe error=%q", got)
|
||||
}
|
||||
longText := strings.Repeat("x", 600)
|
||||
if got := safeDeletionError(errors.New(longText)); len(got) != 450 || strings.Contains(got, "secret") {
|
||||
t.Fatalf("truncated safe error length=%d", len(got))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user