chore: establish local platform baseline

This commit is contained in:
Codex Local
2026-08-26 06:41:54 +08:00
commit 29bdf4900f
575 changed files with 95644 additions and 0 deletions
+271
View File
@@ -0,0 +1,271 @@
package media
import (
"context"
"errors"
"fmt"
"strings"
"time"
"data-terminal-cloud/internal/audit"
"data-terminal-cloud/internal/database"
"data-terminal-cloud/internal/identity"
"github.com/jackc/pgx/v5"
)
type RecordingAccessGrant struct {
ID string `json:"id"`
SubjectID string `json:"subject_id"`
GranteeUserID string `json:"grantee_user_id"`
Permission string `json:"permission"`
Purpose string `json:"purpose"`
GrantedBy string `json:"granted_by"`
GrantedAt time.Time `json:"granted_at"`
ExpiresAt time.Time `json:"expires_at"`
RevokedAt *time.Time `json:"revoked_at,omitempty"`
}
type ConversationMedia struct {
ConversationID string `json:"conversation_id"`
CanPlay bool `json:"can_play"`
CanDownload bool `json:"can_download"`
Segments []Segment `json:"segments"`
}
type MediaAccess struct {
SegmentID string `json:"segment_id"`
Action string `json:"action"`
AccessURL string `json:"access_url"`
AccessUntil time.Time `json:"access_until"`
}
func (s *Service) ListRecordingAccessGrants(ctx context.Context, principal identity.Principal, subjectPublicID string) ([]RecordingAccessGrant, error) {
var tenantID, subjectID, role string
err := s.pool.QueryRow(ctx, `
SELECT sp.tenant_id::text,sp.id::text,hm.role
FROM subject_profile sp
JOIN household_member hm ON hm.household_id=sp.household_id AND hm.tenant_id=sp.tenant_id AND hm.status='active'
JOIN user_account ua ON ua.id=hm.user_id AND ua.oidc_issuer=$2 AND ua.oidc_subject=$3 AND ua.status='active'
WHERE sp.public_id=$1::uuid AND sp.status='active'`, subjectPublicID, principal.Issuer, principal.Subject,
).Scan(&tenantID, &subjectID, &role)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrRecordingAccessDenied
}
if err != nil {
return nil, err
}
if role != "owner" && role != "admin" {
return nil, ErrRecordingAccessDenied
}
rows, err := s.pool.Query(ctx, `
SELECT rag.public_id::text,$3::text,grantee.public_id::text,rag.permission,rag.purpose,
grantor.public_id::text,rag.granted_at,rag.expires_at,rag.revoked_at
FROM recording_access_grant rag
JOIN user_account grantee ON grantee.id=rag.grantee_user_id AND grantee.tenant_id=rag.tenant_id
JOIN user_account grantor ON grantor.id=rag.granted_by AND grantor.tenant_id=rag.tenant_id
WHERE rag.tenant_id=$1::uuid AND rag.subject_id=$2::uuid
ORDER BY rag.granted_at DESC LIMIT 200`, tenantID, subjectID, subjectPublicID)
if err != nil {
return nil, err
}
defer rows.Close()
items := make([]RecordingAccessGrant, 0)
for rows.Next() {
var item RecordingAccessGrant
if err := rows.Scan(&item.ID, &item.SubjectID, &item.GranteeUserID, &item.Permission, &item.Purpose, &item.GrantedBy, &item.GrantedAt, &item.ExpiresAt, &item.RevokedAt); err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
func (s *Service) GrantRecordingAccess(ctx context.Context, principal identity.Principal, subjectPublicID, granteePublicID, permission, purpose, requestID string, expiresAt time.Time) (RecordingAccessGrant, error) {
permission = strings.ToUpper(strings.TrimSpace(permission))
purpose = strings.TrimSpace(purpose)
if (permission != "PLAY" && permission != "DOWNLOAD") || purpose == "" || expiresAt.Before(time.Now().UTC().Add(time.Minute)) {
return RecordingAccessGrant{}, ErrInvalidDeclaration
}
var result RecordingAccessGrant
err := database.Transact(ctx, s.pool, func(tx pgx.Tx) error {
var tenantID, subjectID, requesterID, requesterPublicID, role, granteeID string
err := tx.QueryRow(ctx, `
SELECT sp.tenant_id::text,sp.id::text,ua.id::text,ua.public_id::text,hm.role,grantee.id::text
FROM subject_profile sp
JOIN household_member hm ON hm.household_id=sp.household_id AND hm.tenant_id=sp.tenant_id AND hm.status='active'
JOIN user_account ua ON ua.id=hm.user_id AND ua.oidc_issuer=$2 AND ua.oidc_subject=$3 AND ua.status='active'
JOIN user_account grantee ON grantee.public_id=$4::uuid AND grantee.tenant_id=sp.tenant_id AND grantee.status='active'
JOIN household_member ghm ON ghm.household_id=sp.household_id AND ghm.tenant_id=sp.tenant_id AND ghm.user_id=grantee.id AND ghm.status='active'
WHERE sp.public_id=$1::uuid AND sp.status='active' FOR UPDATE OF sp`, subjectPublicID, principal.Issuer, principal.Subject, granteePublicID,
).Scan(&tenantID, &subjectID, &requesterID, &requesterPublicID, &role, &granteeID)
if errors.Is(err, pgx.ErrNoRows) {
return ErrRecordingAccessDenied
}
if err != nil {
return err
}
if role != "owner" && role != "admin" {
return ErrRecordingAccessDenied
}
_, err = tx.Exec(ctx, `UPDATE recording_access_grant SET revoked_at=now(),revoked_by=$1::uuid,updated_at=now() WHERE tenant_id=$2::uuid AND subject_id=$3::uuid AND grantee_user_id=$4::uuid AND permission=$5 AND revoked_at IS NULL AND expires_at>now()`, requesterID, tenantID, subjectID, granteeID, permission)
if err != nil {
return err
}
err = tx.QueryRow(ctx, `
INSERT INTO recording_access_grant(tenant_id,subject_id,grantee_user_id,permission,purpose,granted_by,expires_at)
VALUES($1::uuid,$2::uuid,$3::uuid,$4,$5,$6::uuid,$7)
RETURNING public_id::text,$8::text,$9::text,$4,$5,$10::text,granted_at,expires_at,revoked_at`, tenantID, subjectID, granteeID, permission, purpose, requesterID, expiresAt, subjectPublicID, granteePublicID, requesterPublicID,
).Scan(&result.ID, &result.SubjectID, &result.GranteeUserID, &result.Permission, &result.Purpose, &result.GrantedBy, &result.GrantedAt, &result.ExpiresAt, &result.RevokedAt)
if err != nil {
return err
}
return s.audit.Record(ctx, tx, audit.Event{TenantID: &tenantID, ActorType: "user", ActorPublicID: &requesterPublicID, Action: "recording.access.grant", ResourceType: "recording_access_grant", ResourceID: &result.ID, RequestID: requestID, Purpose: purpose, Result: "success", Detail: map[string]any{"subject_id": subjectPublicID, "grantee_user_id": granteePublicID, "permission": permission, "expires_at": expiresAt}})
})
return result, err
}
func (s *Service) RevokeRecordingAccess(ctx context.Context, principal identity.Principal, grantPublicID, requestID string) (RecordingAccessGrant, error) {
var result RecordingAccessGrant
err := database.Transact(ctx, s.pool, func(tx pgx.Tx) error {
var tenantID, requesterID, requesterPublicID, role string
err := tx.QueryRow(ctx, `
SELECT rag.tenant_id::text,ua.id::text,ua.public_id::text,hm.role,rag.public_id::text,sp.public_id::text,grantee.public_id::text,
rag.permission,rag.purpose,grantor.public_id::text,rag.granted_at,rag.expires_at,rag.revoked_at
FROM recording_access_grant rag
JOIN subject_profile sp ON sp.id=rag.subject_id AND sp.tenant_id=rag.tenant_id
JOIN household_member hm ON hm.household_id=sp.household_id AND hm.tenant_id=sp.tenant_id AND hm.status='active'
JOIN user_account ua ON ua.id=hm.user_id AND ua.oidc_issuer=$2 AND ua.oidc_subject=$3 AND ua.status='active'
JOIN user_account grantee ON grantee.id=rag.grantee_user_id
JOIN user_account grantor ON grantor.id=rag.granted_by
WHERE rag.public_id=$1::uuid FOR UPDATE OF rag`, grantPublicID, principal.Issuer, principal.Subject,
).Scan(&tenantID, &requesterID, &requesterPublicID, &role, &result.ID, &result.SubjectID, &result.GranteeUserID, &result.Permission, &result.Purpose, &result.GrantedBy, &result.GrantedAt, &result.ExpiresAt, &result.RevokedAt)
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return err
}
if role != "owner" && role != "admin" {
return ErrRecordingAccessDenied
}
if result.RevokedAt == nil {
if err := tx.QueryRow(ctx, `UPDATE recording_access_grant SET revoked_at=now(),revoked_by=$2::uuid,updated_at=now() WHERE public_id=$1::uuid RETURNING revoked_at`, grantPublicID, requesterID).Scan(&result.RevokedAt); err != nil {
return err
}
}
return s.audit.Record(ctx, tx, audit.Event{TenantID: &tenantID, ActorType: "user", ActorPublicID: &requesterPublicID, Action: "recording.access.revoke", ResourceType: "recording_access_grant", ResourceID: &result.ID, RequestID: requestID, Result: "success"})
})
return result, err
}
func (s *Service) GetConversationMedia(ctx context.Context, principal identity.Principal, conversationPublicID string) (ConversationMedia, error) {
var result ConversationMedia
var tenantID, subjectID, userID string
err := s.pool.QueryRow(ctx, `
SELECT c.tenant_id::text,c.subject_id::text,ua.id::text,c.public_id::text,
EXISTS(SELECT 1 FROM recording_access_grant rag WHERE rag.tenant_id=c.tenant_id AND rag.subject_id=c.subject_id AND rag.grantee_user_id=ua.id AND rag.permission='PLAY' AND rag.revoked_at IS NULL AND rag.expires_at>now()),
EXISTS(SELECT 1 FROM recording_access_grant rag WHERE rag.tenant_id=c.tenant_id AND rag.subject_id=c.subject_id AND rag.grantee_user_id=ua.id AND rag.permission='DOWNLOAD' AND rag.revoked_at IS NULL AND rag.expires_at>now())
FROM conversation c
JOIN household_member hm ON hm.household_id=c.household_id AND hm.tenant_id=c.tenant_id AND hm.status='active'
JOIN user_account ua ON ua.id=hm.user_id AND ua.oidc_issuer=$2 AND ua.oidc_subject=$3 AND ua.status='active'
WHERE c.public_id=$1::uuid`, conversationPublicID, principal.Issuer, principal.Subject,
).Scan(&tenantID, &subjectID, &userID, &result.ConversationID, &result.CanPlay, &result.CanDownload)
if errors.Is(err, pgx.ErrNoRows) {
return ConversationMedia{}, ErrNotFound
}
if err != nil {
return ConversationMedia{}, err
}
if !result.CanPlay && !result.CanDownload {
return ConversationMedia{}, ErrRecordingAccessDenied
}
var currentConsent bool
if err := s.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM consent_record WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND consent_type='raw_audio_storage' AND withdrawn_at IS NULL AND (expires_at IS NULL OR expires_at>now()))`, tenantID, subjectID).Scan(&currentConsent); err != nil {
return ConversationMedia{}, err
}
if !currentConsent {
return ConversationMedia{}, ErrRecordingAccessDenied
}
rows, err := s.pool.Query(ctx, `SELECT public_id::text,track,sequence_no,codec,content_type,duration_ms,expected_size,expected_digest,status,COALESCE(last_error_code,''),verified_at,local_delete_after FROM media_segment WHERE tenant_id=$1::uuid AND conversation_id=(SELECT id FROM conversation WHERE public_id=$2::uuid) AND status='VERIFIED' ORDER BY track,sequence_no`, tenantID, conversationPublicID)
if err != nil {
return ConversationMedia{}, err
}
defer rows.Close()
for rows.Next() {
var item Segment
if err := rows.Scan(&item.ID, &item.Track, &item.SequenceNo, &item.Codec, &item.ContentType, &item.DurationMS, &item.ExpectedSize, &item.ExpectedDigest, &item.Status, &item.LastErrorCode, &item.VerifiedAt, &item.LocalDeleteAfter); err != nil {
return ConversationMedia{}, err
}
result.Segments = append(result.Segments, item)
}
_ = userID
return result, rows.Err()
}
func (s *Service) CreateMediaAccess(ctx context.Context, principal identity.Principal, segmentPublicID, action, purpose, requestID string) (MediaAccess, error) {
action = strings.ToUpper(strings.TrimSpace(action))
purpose = strings.TrimSpace(purpose)
if (action != "PLAY" && action != "DOWNLOAD") || purpose == "" {
return MediaAccess{}, ErrInvalidDeclaration
}
var result MediaAccess
denied := false
err := database.Transact(ctx, s.pool, func(tx pgx.Tx) error {
var tenantID, userID, userPublicID, conversationID, conversationPublicID, segmentID, objectRef, contentType, track string
var sequenceNo int
var allowed, currentConsent bool
err := tx.QueryRow(ctx, `
SELECT ms.tenant_id::text,ua.id::text,ua.public_id::text,c.id::text,c.public_id::text,ms.id::text,ms.object_ref,ms.content_type,ms.track,ms.sequence_no,
(hm.id IS NOT NULL AND EXISTS(SELECT 1 FROM recording_access_grant rag WHERE rag.tenant_id=ms.tenant_id AND rag.subject_id=c.subject_id AND rag.grantee_user_id=ua.id AND rag.permission=$4 AND rag.revoked_at IS NULL AND rag.expires_at>now())),
EXISTS(SELECT 1 FROM consent_record cr WHERE cr.tenant_id=ms.tenant_id AND cr.subject_id=c.subject_id AND cr.consent_type='raw_audio_storage' AND cr.withdrawn_at IS NULL AND (cr.expires_at IS NULL OR cr.expires_at>now()))
FROM media_segment ms JOIN conversation c ON c.id=ms.conversation_id
JOIN user_account ua ON ua.tenant_id=ms.tenant_id AND ua.oidc_issuer=$2 AND ua.oidc_subject=$3 AND ua.status='active'
LEFT JOIN household_member hm ON hm.household_id=c.household_id AND hm.tenant_id=c.tenant_id AND hm.user_id=ua.id AND hm.status='active'
WHERE ms.public_id=$1::uuid AND ms.status='VERIFIED'`, segmentPublicID, principal.Issuer, principal.Subject, action,
).Scan(&tenantID, &userID, &userPublicID, &conversationID, &conversationPublicID, &segmentID, &objectRef, &contentType, &track, &sequenceNo, &allowed, &currentConsent)
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return err
}
if !allowed || !currentConsent {
denied = true
if _, err := tx.Exec(ctx, `INSERT INTO media_access_log(tenant_id,actor_user_id,conversation_id,media_segment_id,action,purpose,result,reason_code,request_id) VALUES($1::uuid,$2::uuid,$3::uuid,$4::uuid,$5,$6,'DENIED','ACCESS_GRANT_REQUIRED',$7)`, tenantID, userID, conversationID, segmentID, action, purpose, requestID); err != nil {
return err
}
return s.audit.Record(ctx, tx, audit.Event{TenantID: &tenantID, ActorType: "user", ActorPublicID: &userPublicID, Action: "recording.access", ResourceType: "media_segment", ResourceID: &segmentPublicID, RequestID: requestID, Purpose: purpose, Result: "denied", Detail: map[string]any{"action": action}})
}
extension := ".bin"
if contentType == "audio/wav" {
extension = ".wav"
} else if contentType == "audio/opus" {
extension = ".opus"
} else if contentType == "audio/ogg" {
extension = ".ogg"
}
filename := fmt.Sprintf("%s-%s-%08d%s", conversationPublicID, track, sequenceNo, extension)
until := time.Now().UTC().Add(s.accessTTL)
url, err := s.store.PresignMediaDownload(ctx, objectRef, s.accessTTL, filename, contentType, action == "PLAY")
if err != nil {
return err
}
if _, err := tx.Exec(ctx, `INSERT INTO media_access_log(tenant_id,actor_user_id,conversation_id,media_segment_id,action,purpose,result,request_id) VALUES($1::uuid,$2::uuid,$3::uuid,$4::uuid,$5,$6,'SUCCESS',$7)`, tenantID, userID, conversationID, segmentID, action, purpose, requestID); err != nil {
return err
}
if err := s.audit.Record(ctx, tx, audit.Event{TenantID: &tenantID, ActorType: "user", ActorPublicID: &userPublicID, Action: "recording.access", ResourceType: "media_segment", ResourceID: &segmentPublicID, RequestID: requestID, Purpose: purpose, Result: "success", Detail: map[string]any{"action": action}}); err != nil {
return err
}
result = MediaAccess{SegmentID: segmentPublicID, Action: action, AccessURL: url, AccessUntil: until}
return nil
})
if err != nil {
return MediaAccess{}, err
}
if denied {
return MediaAccess{}, ErrRecordingAccessDenied
}
return result, nil
}
+90
View File
@@ -0,0 +1,90 @@
package media
import (
"context"
"errors"
"strings"
"time"
"data-terminal-cloud/internal/audit"
"data-terminal-cloud/internal/database"
"data-terminal-cloud/internal/deviceauth"
"github.com/jackc/pgx/v5"
)
// AcknowledgeLocalDelete records only deletion of the device-side encrypted copy.
// The verified cloud object and its media_segment lifecycle remain unchanged.
func (s *Service) AcknowledgeLocalDelete(ctx context.Context, identity deviceauth.Identity, segmentPublicID, status, errorCode, requestID string) (Segment, error) {
status = strings.ToUpper(strings.TrimSpace(status))
errorCode = strings.ToUpper(strings.TrimSpace(errorCode))
if status != "DELETED" && status != "FAILED" {
return Segment{}, ErrInvalidDeclaration
}
if status == "DELETED" && errorCode != "" {
return Segment{}, ErrInvalidDeclaration
}
if status == "FAILED" && (errorCode == "" || len(errorCode) > 80) {
return Segment{}, ErrInvalidDeclaration
}
var result Segment
err := database.Transact(ctx, s.pool, func(tx pgx.Tx) error {
var cloudStatus, storedStatus string
var deleteAfter *time.Time
err := tx.QueryRow(ctx, `
SELECT status,COALESCE(local_delete_status,''),local_delete_after
FROM media_segment
WHERE public_id=$1::uuid AND tenant_id=$2::uuid AND device_id=$3::uuid
FOR UPDATE`, segmentPublicID, identity.TenantID, identity.ID,
).Scan(&cloudStatus, &storedStatus, &deleteAfter)
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return err
}
if cloudStatus != "VERIFIED" || deleteAfter == nil {
return ErrConflict
}
if storedStatus == "DELETED" {
return scanDeviceSegment(ctx, tx, identity, segmentPublicID, &result)
}
if status == "DELETED" && time.Now().UTC().Before(*deleteAfter) {
return ErrLocalDeleteTooEarly
}
if _, err := tx.Exec(ctx, `
UPDATE media_segment
SET local_delete_status=$4,local_delete_reported_at=now(),local_delete_error_code=NULLIF($5,''),updated_at=now()
WHERE public_id=$1::uuid AND tenant_id=$2::uuid AND device_id=$3::uuid`,
segmentPublicID, identity.TenantID, identity.ID, status, errorCode,
); err != nil {
return err
}
auditResult := "success"
if status == "FAILED" {
auditResult = "failure"
}
if err := s.audit.Record(ctx, tx, audit.Event{
TenantID: &identity.TenantID, ActorType: "device", ActorPublicID: &identity.PublicID,
Action: "media.local_delete.ack", ResourceType: "media_segment", ResourceID: &segmentPublicID,
RequestID: requestID, Result: auditResult, Detail: map[string]any{"status": status, "error_code": errorCode},
}); err != nil {
return err
}
return scanDeviceSegment(ctx, tx, identity, segmentPublicID, &result)
})
return result, err
}
func scanDeviceSegment(ctx context.Context, tx pgx.Tx, identity deviceauth.Identity, segmentPublicID string, result *Segment) error {
return tx.QueryRow(ctx, `
SELECT public_id::text,track,sequence_no,codec,content_type,duration_ms,expected_size,expected_digest,status,
COALESCE(last_error_code,''),verified_at,local_delete_after,COALESCE(local_delete_status,''),
local_delete_reported_at,COALESCE(local_delete_error_code,'')
FROM media_segment WHERE public_id=$1::uuid AND tenant_id=$2::uuid AND device_id=$3::uuid`,
segmentPublicID, identity.TenantID, identity.ID,
).Scan(&result.ID, &result.Track, &result.SequenceNo, &result.Codec, &result.ContentType, &result.DurationMS,
&result.ExpectedSize, &result.ExpectedDigest, &result.Status, &result.LastErrorCode, &result.VerifiedAt,
&result.LocalDeleteAfter, &result.LocalDeleteStatus, &result.LocalDeleteReportedAt, &result.LocalDeleteErrorCode)
}
+154
View File
@@ -0,0 +1,154 @@
package media
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"data-terminal-cloud/internal/audit"
"data-terminal-cloud/internal/database"
"data-terminal-cloud/internal/objectstore"
"data-terminal-cloud/internal/outbox"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
type verificationStore interface {
VerifyDigest(context.Context, string, string, int64) error
ReadPrefix(context.Context, string, int64) ([]byte, error)
}
type Processor struct {
pool *pgxpool.Pool
store verificationStore
audit *audit.Repository
localDeleteGrace time.Duration
}
func NewProcessor(pool *pgxpool.Pool, store verificationStore, auditRepository *audit.Repository, localDeleteGrace ...time.Duration) (*Processor, error) {
if pool == nil || store == nil || auditRepository == nil {
return nil, errors.New("media processor requires database, object store and audit repository")
}
grace := time.Hour
if len(localDeleteGrace) > 0 {
grace = localDeleteGrace[0]
}
if grace <= 0 {
return nil, errors.New("media local delete grace must be positive")
}
return &Processor{pool: pool, store: store, audit: auditRepository, localDeleteGrace: grace}, nil
}
func (p *Processor) Handle(ctx context.Context, event outbox.Event) error {
var payload struct {
BatchID string `json:"batch_id"`
}
if err := json.Unmarshal(event.Payload, &payload); err != nil || payload.BatchID == "" {
return errors.New("media verification event has no batch_id")
}
var batchID, tenantID, status string
err := p.pool.QueryRow(ctx, `SELECT id::text,tenant_id::text,status FROM upload_batch WHERE public_id=$1::uuid`, payload.BatchID).Scan(&batchID, &tenantID, &status)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
if status == "COMPLETED" || status == "FAILED" || status == "EXPIRED" {
return nil
}
if status != "VERIFYING" {
return fmt.Errorf("media batch %s is in unexpected status %s", payload.BatchID, status)
}
rows, err := p.pool.Query(ctx, `
SELECT ms.id::text,ms.object_ref,ms.expected_digest,ms.expected_size,ms.codec,ms.duration_ms
FROM media_segment ms JOIN upload_batch_segment ubs ON ubs.media_segment_id=ms.id
WHERE ubs.upload_batch_id=$1::uuid AND ms.status='VERIFYING'
ORDER BY ms.track,ms.sequence_no`, batchID)
if err != nil {
return err
}
type pendingSegment struct {
id, objectRef, digest, codec string
size int64
durationMS int
}
var pending []pendingSegment
for rows.Next() {
var item pendingSegment
if err := rows.Scan(&item.id, &item.objectRef, &item.digest, &item.size, &item.codec, &item.durationMS); err != nil {
rows.Close()
return err
}
pending = append(pending, item)
}
rows.Close()
if err := rows.Err(); err != nil {
return err
}
for _, segment := range pending {
err := p.store.VerifyDigest(ctx, segment.objectRef, segment.digest, segment.size)
if err == nil {
prefix, readErr := p.store.ReadPrefix(ctx, segment.objectRef, 64*1024)
if readErr != nil {
_, _ = p.pool.Exec(ctx, `UPDATE upload_batch SET verification_attempts=verification_attempts+1,last_error_code='OBJECT_NOT_VISIBLE',updated_at=now() WHERE id=$1::uuid`, batchID)
return outbox.Retryable(fmt.Errorf("read media segment header: %w", readErr))
}
validationCode := validateAudio(segment.codec, segment.durationMS, prefix)
if validationCode != "" {
if _, updateErr := p.pool.Exec(ctx, `UPDATE media_segment SET status='REJECTED',verification_attempts=verification_attempts+1,last_error_code=$2,updated_at=now() WHERE id=$1::uuid AND status='VERIFYING'`, segment.id, validationCode); updateErr != nil {
return updateErr
}
continue
}
if _, updateErr := p.pool.Exec(ctx, `UPDATE media_segment SET status='VERIFIED',verification_attempts=verification_attempts+1,verified_at=now(),local_delete_after=now()+($2::bigint*interval '1 microsecond'),last_error_code=NULL,updated_at=now() WHERE id=$1::uuid AND status='VERIFYING'`, segment.id, p.localDeleteGrace.Microseconds()); updateErr != nil {
return updateErr
}
continue
}
if errors.Is(err, objectstore.ErrObjectSizeMismatch) || errors.Is(err, objectstore.ErrObjectDigestMismatch) {
code := "DIGEST_MISMATCH"
if errors.Is(err, objectstore.ErrObjectSizeMismatch) {
code = "SIZE_MISMATCH"
}
if _, updateErr := p.pool.Exec(ctx, `UPDATE media_segment SET status='REJECTED',verification_attempts=verification_attempts+1,last_error_code=$2,updated_at=now() WHERE id=$1::uuid AND status='VERIFYING'`, segment.id, code); updateErr != nil {
return updateErr
}
continue
}
_, _ = p.pool.Exec(ctx, `UPDATE upload_batch SET verification_attempts=verification_attempts+1,last_error_code='OBJECT_NOT_VISIBLE',updated_at=now() WHERE id=$1::uuid`, batchID)
return outbox.Retryable(fmt.Errorf("verify media segment object: %w", err))
}
return database.Transact(ctx, p.pool, func(tx pgx.Tx) error {
var verified, rejected, remaining int
if err := tx.QueryRow(ctx, `
SELECT count(*) FILTER (WHERE ms.status='VERIFIED'),count(*) FILTER (WHERE ms.status='REJECTED'),count(*) FILTER (WHERE ms.status NOT IN ('VERIFIED','REJECTED','DELETED'))
FROM media_segment ms JOIN upload_batch_segment ubs ON ubs.media_segment_id=ms.id WHERE ubs.upload_batch_id=$1::uuid`, batchID,
).Scan(&verified, &rejected, &remaining); err != nil {
return err
}
if remaining > 0 {
return fmt.Errorf("media batch still has %d non-terminal segments", remaining)
}
batchStatus := "COMPLETED"
lastError := any(nil)
if rejected > 0 {
batchStatus = "FAILED"
lastError = "SEGMENT_REJECTED"
}
if _, err := tx.Exec(ctx, `UPDATE upload_batch SET status=$2,verified_count=$3,rejected_count=$4,verification_attempts=verification_attempts+1,last_error_code=$5,completed_at=now(),updated_at=now() WHERE id=$1::uuid`, batchID, batchStatus, verified, rejected, lastError); err != nil {
return err
}
auditResult := "success"
if rejected > 0 {
auditResult = "failure"
}
return p.audit.Record(ctx, tx, audit.Event{TenantID: &tenantID, ActorType: "worker", Action: "media.upload_batch.verify", ResourceType: "upload_batch", ResourceID: &payload.BatchID, Result: auditResult, Detail: map[string]any{"status": batchStatus, "verified": verified, "rejected": rejected}})
})
}
+350
View File
@@ -0,0 +1,350 @@
package media
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"data-terminal-cloud/internal/audit"
"data-terminal-cloud/internal/database"
"data-terminal-cloud/internal/deviceauth"
"data-terminal-cloud/internal/outbox"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
var (
ErrAccessDenied = errors.New("media upload access denied")
ErrNotFound = errors.New("media upload batch not found")
ErrConflict = errors.New("media upload request conflicts with existing data")
ErrExpired = errors.New("media upload batch expired")
ErrInvalidDeclaration = errors.New("media segment declaration is invalid")
ErrRecordingAccessDenied = errors.New("recording access denied")
ErrLocalDeleteTooEarly = errors.New("media local deletion is not due")
)
type objectStore interface {
PresignUpload(context.Context, string, time.Duration) (string, error)
PresignMediaDownload(context.Context, string, time.Duration, string, string, bool) (string, error)
}
type SegmentDeclaration struct {
Track string `json:"track"`
SequenceNo int `json:"sequence_no"`
Codec string `json:"codec"`
ContentType string `json:"content_type"`
DurationMS int `json:"duration_ms"`
ExpectedSize int64 `json:"expected_size"`
ExpectedDigest string `json:"expected_digest"`
}
type Segment struct {
ID string `json:"id"`
Track string `json:"track"`
SequenceNo int `json:"sequence_no"`
Codec string `json:"codec"`
ContentType string `json:"content_type"`
DurationMS int `json:"duration_ms"`
ExpectedSize int64 `json:"expected_size"`
ExpectedDigest string `json:"expected_digest"`
Status string `json:"status"`
LastErrorCode string `json:"last_error_code,omitempty"`
UploadURL string `json:"upload_url,omitempty"`
UploadUntil *time.Time `json:"upload_until,omitempty"`
VerifiedAt *time.Time `json:"verified_at,omitempty"`
LocalDeleteAfter *time.Time `json:"local_delete_after,omitempty"`
LocalDeleteStatus string `json:"local_delete_status,omitempty"`
LocalDeleteReportedAt *time.Time `json:"local_delete_reported_at,omitempty"`
LocalDeleteErrorCode string `json:"local_delete_error_code,omitempty"`
}
type UploadBatch struct {
ID string `json:"id"`
ConversationID string `json:"conversation_id"`
Status string `json:"status"`
SegmentCount int `json:"segment_count"`
ExpectedTotalSize int64 `json:"expected_total_size"`
VerifiedCount int `json:"verified_count"`
RejectedCount int `json:"rejected_count"`
VerificationAttempts int `json:"verification_attempts"`
LastErrorCode string `json:"last_error_code,omitempty"`
ExpiresAt time.Time `json:"expires_at"`
Segments []Segment `json:"segments"`
}
type Service struct {
pool *pgxpool.Pool
outbox *outbox.Store
audit *audit.Repository
store objectStore
uploadTTL time.Duration
accessTTL time.Duration
}
func NewService(pool *pgxpool.Pool, outboxStore *outbox.Store, auditRepository *audit.Repository, store objectStore, uploadTTL time.Duration) (*Service, error) {
if pool == nil || outboxStore == nil || auditRepository == nil || store == nil || uploadTTL < 10*time.Minute || uploadTTL > 20*time.Minute {
return nil, errors.New("media service requires database, outbox, audit, object store and a 10-20 minute upload TTL")
}
return &Service{pool: pool, outbox: outboxStore, audit: auditRepository, store: store, uploadTTL: uploadTTL, accessTTL: 5 * time.Minute}, nil
}
func (s *Service) PrepareUploadBatch(ctx context.Context, identity deviceauth.Identity, conversationPublicID, idempotencyKey, requestID string, declarations []SegmentDeclaration) (UploadBatch, error) {
manifestDigest, normalized, totalSize, err := normalizeManifest(declarations)
if err != nil {
return UploadBatch{}, err
}
var batchPublicID string
err = database.Transact(ctx, s.pool, func(tx pgx.Tx) error {
var conversationID, retentionMode, conversationStatus string
var currentConsent bool
err := tx.QueryRow(ctx, `
SELECT c.id::text,c.retention_mode,c.status,
EXISTS(
SELECT 1 FROM consent_record cr
WHERE cr.tenant_id=c.tenant_id AND cr.subject_id=c.subject_id
AND cr.consent_type='raw_audio_storage' AND cr.withdrawn_at IS NULL
AND (cr.expires_at IS NULL OR cr.expires_at>now())
)
FROM conversation c
WHERE c.public_id=$1::uuid AND c.tenant_id=$2::uuid AND c.device_id=$3::uuid`,
conversationPublicID, identity.TenantID, identity.ID,
).Scan(&conversationID, &retentionMode, &conversationStatus, &currentConsent)
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return err
}
if retentionMode != "transcript_and_audio" || !currentConsent || (conversationStatus != "COMPLETED" && conversationStatus != "FAILED") {
return ErrAccessDenied
}
var storedDigest string
err = tx.QueryRow(ctx, `SELECT public_id::text,manifest_digest FROM upload_batch WHERE device_id=$1::uuid AND conversation_id=$2::uuid AND idempotency_key=$3 FOR UPDATE`, identity.ID, conversationID, idempotencyKey).Scan(&batchPublicID, &storedDigest)
if err == nil {
if storedDigest != manifestDigest {
return ErrConflict
}
return nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return err
}
var batchID string
expiresAt := time.Now().UTC().Add(s.uploadTTL)
err = tx.QueryRow(ctx, `
INSERT INTO upload_batch(tenant_id,conversation_id,device_id,idempotency_key,manifest_digest,status,segment_count,expected_total_size,expires_at)
VALUES($1::uuid,$2::uuid,$3::uuid,$4,$5,'PENDING',$6,$7,$8)
RETURNING id::text,public_id::text`, identity.TenantID, conversationID, identity.ID, idempotencyKey, manifestDigest, len(normalized), totalSize, expiresAt,
).Scan(&batchID, &batchPublicID)
if err != nil {
return err
}
tenantHash := sha256.Sum256([]byte(identity.TenantID))
type manifestRow struct {
Track string `json:"track"`
SequenceNo int `json:"sequence_no"`
Codec string `json:"codec"`
ContentType string `json:"content_type"`
DurationMS int `json:"duration_ms"`
ExpectedSize int64 `json:"expected_size"`
ExpectedDigest string `json:"expected_digest"`
ObjectRef string `json:"object_ref"`
}
manifestRows := make([]manifestRow, 0, len(normalized))
for _, declaration := range normalized {
manifestRows = append(manifestRows, manifestRow{
Track: declaration.Track, SequenceNo: declaration.SequenceNo, Codec: declaration.Codec,
ContentType: declaration.ContentType, DurationMS: declaration.DurationMS,
ExpectedSize: declaration.ExpectedSize, ExpectedDigest: declaration.ExpectedDigest,
ObjectRef: fmt.Sprintf("tenants/%s/media/%s/%s/%08d-%s.bin", hex.EncodeToString(tenantHash[:16]), conversationPublicID, declaration.Track, declaration.SequenceNo, declaration.ExpectedDigest[:16]),
})
}
manifestJSON, err := json.Marshal(manifestRows)
if err != nil {
return err
}
if _, err = tx.Exec(ctx, `
INSERT INTO media_segment(tenant_id,conversation_id,device_id,track,sequence_no,codec,content_type,duration_ms,expected_size,expected_digest,object_ref,status)
SELECT $1::uuid,$2::uuid,$3::uuid,x.track,x.sequence_no,x.codec,x.content_type,x.duration_ms,x.expected_size,x.expected_digest,x.object_ref,'AUTHORIZED'
FROM jsonb_to_recordset($4::jsonb) AS x(track text,sequence_no integer,codec text,content_type text,duration_ms integer,expected_size bigint,expected_digest text,object_ref text)
ON CONFLICT (conversation_id,track,sequence_no) DO NOTHING`, identity.TenantID, conversationID, identity.ID, manifestJSON); err != nil {
return err
}
var foundCount, validCount int
if err = tx.QueryRow(ctx, `
SELECT count(*),count(*) FILTER(WHERE ms.codec=x.codec AND ms.content_type=x.content_type AND ms.duration_ms=x.duration_ms
AND ms.expected_size=x.expected_size AND ms.expected_digest=x.expected_digest AND ms.status<>'DELETED')
FROM jsonb_to_recordset($2::jsonb) AS x(track text,sequence_no integer,codec text,content_type text,duration_ms integer,expected_size bigint,expected_digest text)
JOIN media_segment ms ON ms.conversation_id=$1::uuid AND ms.track=x.track AND ms.sequence_no=x.sequence_no`, conversationID, manifestJSON).Scan(&foundCount, &validCount); err != nil {
return err
}
if foundCount != len(normalized) || validCount != len(normalized) {
return ErrConflict
}
if _, err = tx.Exec(ctx, `
UPDATE media_segment ms SET status='AUTHORIZED',last_error_code=NULL,updated_at=now()
FROM jsonb_to_recordset($2::jsonb) AS x(track text,sequence_no integer)
WHERE ms.conversation_id=$1::uuid AND ms.track=x.track AND ms.sequence_no=x.sequence_no AND ms.status IN('REJECTED','DECLARED')`, conversationID, manifestJSON); err != nil {
return err
}
linkTag, err := tx.Exec(ctx, `
INSERT INTO upload_batch_segment(upload_batch_id,media_segment_id,tenant_id)
SELECT $1::uuid,ms.id,$2::uuid
FROM jsonb_to_recordset($4::jsonb) AS x(track text,sequence_no integer)
JOIN media_segment ms ON ms.conversation_id=$3::uuid AND ms.track=x.track AND ms.sequence_no=x.sequence_no`, batchID, identity.TenantID, conversationID, manifestJSON)
if err != nil {
return err
}
if linkTag.RowsAffected() != int64(len(normalized)) {
return ErrConflict
}
if _, err = tx.Exec(ctx, `UPDATE upload_batch SET status='UPLOADING',updated_at=now() WHERE id=$1::uuid`, batchID); err != nil {
return err
}
return s.audit.Record(ctx, tx, audit.Event{TenantID: &identity.TenantID, ActorType: "device", ActorPublicID: &identity.PublicID, Action: "media.upload_batch.prepare", ResourceType: "upload_batch", ResourceID: &batchPublicID, RequestID: requestID, Result: "success", Detail: map[string]any{"conversation_id": conversationPublicID, "segment_count": len(normalized)}})
})
if err != nil {
return UploadBatch{}, err
}
return s.GetUploadBatch(ctx, identity, batchPublicID, true)
}
func (s *Service) CompleteUploadBatch(ctx context.Context, identity deviceauth.Identity, batchPublicID, requestID string) (UploadBatch, error) {
err := database.Transact(ctx, s.pool, func(tx pgx.Tx) error {
var batchID, status string
var verificationAttempts int
var expiresAt time.Time
err := tx.QueryRow(ctx, `SELECT id::text,status,verification_attempts,expires_at FROM upload_batch WHERE public_id=$1::uuid AND tenant_id=$2::uuid AND device_id=$3::uuid FOR UPDATE`, batchPublicID, identity.TenantID, identity.ID).Scan(&batchID, &status, &verificationAttempts, &expiresAt)
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return err
}
if status == "COMPLETED" || status == "VERIFYING" {
return nil
}
if time.Now().UTC().After(expiresAt) || status == "EXPIRED" {
return ErrExpired
}
if status != "UPLOADING" && status != "FAILED" {
return ErrConflict
}
if _, err = tx.Exec(ctx, `UPDATE media_segment ms SET status='VERIFYING',uploaded_at=COALESCE(uploaded_at,now()),last_error_code=NULL,updated_at=now() FROM upload_batch_segment ubs WHERE ubs.upload_batch_id=$1::uuid AND ubs.media_segment_id=ms.id AND ms.status IN ('AUTHORIZED','REJECTED')`, batchID); err != nil {
return err
}
if _, err = tx.Exec(ctx, `UPDATE upload_batch SET status='VERIFYING',last_error_code=NULL,updated_at=now() WHERE id=$1::uuid`, batchID); err != nil {
return err
}
payload, _ := json.Marshal(map[string]string{"batch_id": batchPublicID})
deduplicationKey := fmt.Sprintf("media-verify:%s:%d", batchPublicID, verificationAttempts+1)
if _, err = s.outbox.Enqueue(ctx, tx, outbox.NewEvent{TenantID: &identity.TenantID, AggregateType: "upload_batch", AggregateID: batchPublicID, EventType: "media.verify_requested", DeduplicationKey: &deduplicationKey, Payload: payload}); err != nil {
return err
}
return s.audit.Record(ctx, tx, audit.Event{TenantID: &identity.TenantID, ActorType: "device", ActorPublicID: &identity.PublicID, Action: "media.upload_batch.complete", ResourceType: "upload_batch", ResourceID: &batchPublicID, RequestID: requestID, Result: "success", Detail: map[string]any{"status": "VERIFYING"}})
})
if err != nil {
return UploadBatch{}, err
}
return s.GetUploadBatch(ctx, identity, batchPublicID, false)
}
func (s *Service) GetUploadBatch(ctx context.Context, identity deviceauth.Identity, batchPublicID string, includeUploadURLs bool) (UploadBatch, error) {
var item UploadBatch
err := s.pool.QueryRow(ctx, `
SELECT ub.public_id::text,c.public_id::text,ub.status,ub.segment_count,ub.expected_total_size,
ub.verified_count,ub.rejected_count,ub.verification_attempts,COALESCE(ub.last_error_code,''),ub.expires_at
FROM upload_batch ub JOIN conversation c ON c.id=ub.conversation_id
WHERE ub.public_id=$1::uuid AND ub.tenant_id=$2::uuid AND ub.device_id=$3::uuid`, batchPublicID, identity.TenantID, identity.ID,
).Scan(&item.ID, &item.ConversationID, &item.Status, &item.SegmentCount, &item.ExpectedTotalSize, &item.VerifiedCount, &item.RejectedCount, &item.VerificationAttempts, &item.LastErrorCode, &item.ExpiresAt)
if errors.Is(err, pgx.ErrNoRows) {
return UploadBatch{}, ErrNotFound
}
if err != nil {
return UploadBatch{}, err
}
rows, err := s.pool.Query(ctx, `
SELECT ms.public_id::text,ms.track,ms.sequence_no,ms.codec,ms.content_type,ms.duration_ms,ms.expected_size,
ms.expected_digest,ms.status,COALESCE(ms.last_error_code,''),ms.verified_at,ms.local_delete_after,
COALESCE(ms.local_delete_status,''),ms.local_delete_reported_at,COALESCE(ms.local_delete_error_code,''),ms.object_ref
FROM media_segment ms JOIN upload_batch_segment ubs ON ubs.media_segment_id=ms.id
JOIN upload_batch ub ON ub.id=ubs.upload_batch_id
WHERE ub.public_id=$1::uuid AND ub.tenant_id=$2::uuid ORDER BY ms.track,ms.sequence_no`, batchPublicID, identity.TenantID)
if err != nil {
return UploadBatch{}, err
}
defer rows.Close()
for rows.Next() {
var segment Segment
var objectRef string
if err := rows.Scan(&segment.ID, &segment.Track, &segment.SequenceNo, &segment.Codec, &segment.ContentType, &segment.DurationMS, &segment.ExpectedSize, &segment.ExpectedDigest, &segment.Status, &segment.LastErrorCode, &segment.VerifiedAt, &segment.LocalDeleteAfter, &segment.LocalDeleteStatus, &segment.LocalDeleteReportedAt, &segment.LocalDeleteErrorCode, &objectRef); err != nil {
return UploadBatch{}, err
}
if includeUploadURLs && segment.Status == "AUTHORIZED" && time.Now().UTC().Before(item.ExpiresAt) {
url, signErr := s.store.PresignUpload(ctx, objectRef, time.Until(item.ExpiresAt))
if signErr != nil {
return UploadBatch{}, signErr
}
segment.UploadURL = url
until := item.ExpiresAt
segment.UploadUntil = &until
}
item.Segments = append(item.Segments, segment)
}
return item, rows.Err()
}
func normalizeManifest(declarations []SegmentDeclaration) (string, []SegmentDeclaration, int64, error) {
if len(declarations) == 0 || len(declarations) > 1000 {
return "", nil, 0, ErrInvalidDeclaration
}
normalized := append([]SegmentDeclaration(nil), declarations...)
sort.Slice(normalized, func(i, j int) bool {
if normalized[i].Track == normalized[j].Track {
return normalized[i].SequenceNo < normalized[j].SequenceNo
}
return normalized[i].Track < normalized[j].Track
})
seen := make(map[string]struct{}, len(normalized))
var total int64
for index := range normalized {
item := &normalized[index]
item.Track = strings.ToLower(strings.TrimSpace(item.Track))
item.Codec = strings.ToLower(strings.TrimSpace(item.Codec))
item.ContentType = strings.ToLower(strings.TrimSpace(item.ContentType))
item.ExpectedDigest = strings.ToLower(strings.TrimSpace(item.ExpectedDigest))
key := fmt.Sprintf("%s:%d", item.Track, item.SequenceNo)
_, duplicate := seen[key]
if (item.Track != "near" && item.Track != "far") || item.SequenceNo < 0 || duplicate || !validCodec(item.Codec, item.ContentType) || item.DurationMS < 1 || item.DurationMS > 600000 || item.ExpectedSize < 1 || item.ExpectedSize > 64*1024*1024 || len(item.ExpectedDigest) != 64 {
return "", nil, 0, ErrInvalidDeclaration
}
if _, err := hex.DecodeString(item.ExpectedDigest); err != nil {
return "", nil, 0, ErrInvalidDeclaration
}
seen[key] = struct{}{}
if total > 2*1024*1024*1024-item.ExpectedSize {
return "", nil, 0, ErrInvalidDeclaration
}
total += item.ExpectedSize
}
encoded, _ := json.Marshal(normalized)
digest := sha256.Sum256(encoded)
return hex.EncodeToString(digest[:]), normalized, total, nil
}
func validCodec(codec, contentType string) bool {
allowed := map[string]string{
"wav_pcm_s16le": "audio/wav",
}
return allowed[codec] == contentType
}
+205
View File
@@ -0,0 +1,205 @@
package media
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"os"
"testing"
"time"
"data-terminal-cloud/internal/audit"
"data-terminal-cloud/internal/database"
"data-terminal-cloud/internal/deviceauth"
identitypkg "data-terminal-cloud/internal/identity"
"data-terminal-cloud/internal/orchestration"
"data-terminal-cloud/internal/outbox"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
const (
mediaTestConsentID = "90000000-0000-0000-0000-000000000801"
mediaTestDeviceID = "10000000-0000-0000-0000-000000000501"
mediaTestDevicePub = "10000000-0000-0000-0000-000000000511"
)
type mediaTestStore struct {
wav []byte
}
func (*mediaTestStore) PresignUpload(_ context.Context, reference string, _ time.Duration) (string, error) {
return "https://upload.test/" + reference, nil
}
func (*mediaTestStore) PresignMediaDownload(_ context.Context, reference string, _ time.Duration, _, _ string, _ bool) (string, error) {
return "https://download.test/" + reference, nil
}
func (*mediaTestStore) VerifyDigest(context.Context, string, string, int64) error { return nil }
func (s *mediaTestStore) ReadPrefix(context.Context, string, int64) ([]byte, error) {
return append([]byte(nil), s.wav...), nil
}
func TestUploadVerificationAndLocalDeleteLifecycle(t *testing.T) {
databaseURL := os.Getenv("TEST_DATABASE_URL")
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := database.Open(ctx, databaseURL, "media-service-test")
if err != nil {
t.Fatal(err)
}
defer pool.Close()
cleanupMediaFixture(t, ctx, pool)
defer cleanupMediaFixture(t, context.Background(), pool)
if _, err := pool.Exec(ctx, `INSERT INTO consent_record (
id,public_id,tenant_id,subject_id,consent_type,scope,policy_version,notice_digest,actor_user_id
) VALUES (
$1::uuid,'90000000-0000-0000-0000-000000000802','10000000-0000-0000-0000-000000000001',
'10000000-0000-0000-0000-000000000401','raw_audio_storage','{}'::jsonb,'media-test-v1',repeat('d',64),
'10000000-0000-0000-0000-000000000101'
)`, mediaTestConsentID); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO consent_snapshot (
id,public_id,tenant_id,subject_id,snapshot_version,effective_consents,source_consent_ids
) VALUES (
'90000000-0000-0000-0000-000000000803','90000000-0000-0000-0000-000000000804',
'10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000401',9001,
'{"voice_chat":true,"transcript_storage":true,"raw_audio_storage":true}'::jsonb,
'["90000000-0000-0000-0000-000000000802"]'::jsonb
)`); err != nil {
t.Fatal(err)
}
identity := deviceauth.Identity{ID: mediaTestDeviceID, PublicID: mediaTestDevicePub, TenantID: "10000000-0000-0000-0000-000000000001", Status: "registered"}
tx, err := pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
t.Fatal(err)
}
resume := true
created, internalID, err := orchestration.NewRepository().Create(ctx, tx, identity, orchestration.CreateInput{
DeviceTime: time.Now().UTC(), Capabilities: orchestration.Capabilities{RTC: true, AudioTracks: []string{"near"}, UploadResume: &resume},
Network: orchestration.Network{Type: "test"}, RequestedGoal: "media_integration",
}, orchestration.AllocationOptions{Now: time.Now().UTC(), AllocationExpires: time.Now().UTC().Add(time.Minute), TokenExpires: time.Now().UTC().Add(5 * time.Minute), RTCAppID: "test", EngineMode: "s2s_hybrid", HeartbeatInterval: 15})
if err != nil {
_ = tx.Rollback(ctx)
t.Fatal(err)
}
if err := tx.Commit(ctx); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `UPDATE conversation SET status='COMPLETED',ended_at=now(),retention_mode='transcript_and_audio' WHERE id=$1::uuid`, internalID); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED',lease_expires_at=NULL WHERE conversation_id=$1::uuid`, internalID); err != nil {
t.Fatal(err)
}
wav := syntheticWAV(16000, 1, 16000)
digestValue := sha256.Sum256(wav)
declaration := SegmentDeclaration{Track: "near", SequenceNo: 1, Codec: "wav_pcm_s16le", ContentType: "audio/wav", DurationMS: 1000, ExpectedSize: int64(len(wav)), ExpectedDigest: hex.EncodeToString(digestValue[:])}
store := &mediaTestStore{wav: wav}
service, err := NewService(pool, outbox.NewStore(pool), audit.NewRepository(), store, 15*time.Minute)
if err != nil {
t.Fatal(err)
}
batch, err := service.PrepareUploadBatch(ctx, identity, created.ConversationID, "media-integration-batch", "media-prepare", []SegmentDeclaration{declaration})
if err != nil || batch.Status != "UPLOADING" || len(batch.Segments) != 1 || batch.Segments[0].UploadURL == "" {
t.Fatalf("prepared batch=%#v err=%v", batch, err)
}
replayed, err := service.PrepareUploadBatch(ctx, identity, created.ConversationID, "media-integration-batch", "media-prepare-replay", []SegmentDeclaration{declaration})
if err != nil || replayed.ID != batch.ID {
t.Fatalf("replayed batch=%#v err=%v", replayed, err)
}
verifying, err := service.CompleteUploadBatch(ctx, identity, batch.ID, "media-complete")
if err != nil || verifying.Status != "VERIFYING" {
t.Fatalf("verifying batch=%#v err=%v", verifying, err)
}
processor, err := NewProcessor(pool, store, audit.NewRepository())
if err != nil {
t.Fatal(err)
}
payload, _ := json.Marshal(map[string]string{"batch_id": batch.ID})
if err := processor.Handle(ctx, outbox.Event{Payload: payload}); err != nil {
t.Fatal(err)
}
completed, err := service.GetUploadBatch(ctx, identity, batch.ID, false)
if err != nil || completed.Status != "COMPLETED" || completed.VerifiedCount != 1 || completed.Segments[0].Status != "VERIFIED" {
t.Fatalf("completed batch=%#v err=%v", completed, err)
}
segmentID := completed.Segments[0].ID
owner := identitypkg.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000101"}
editor := identitypkg.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000102"}
grant, err := service.GrantRecordingAccess(ctx, owner, "10000000-0000-0000-0000-000000000411", "10000000-0000-0000-0000-000000000112", "PLAY", "媒体集成测试播放", "media-grant", time.Now().UTC().Add(time.Hour))
if err != nil || grant.Permission != "PLAY" {
t.Fatalf("recording grant=%#v err=%v", grant, err)
}
grants, err := service.ListRecordingAccessGrants(ctx, owner, "10000000-0000-0000-0000-000000000411")
if err != nil || len(grants) != 1 || grants[0].ID != grant.ID {
t.Fatalf("recording grants=%#v err=%v", grants, err)
}
conversationMedia, err := service.GetConversationMedia(ctx, editor, created.ConversationID)
if err != nil || !conversationMedia.CanPlay || conversationMedia.CanDownload || len(conversationMedia.Segments) != 1 {
t.Fatalf("conversation media=%#v err=%v", conversationMedia, err)
}
access, err := service.CreateMediaAccess(ctx, editor, segmentID, "PLAY", "媒体集成测试试听", "media-access-play")
if err != nil || access.Action != "PLAY" || access.AccessURL == "" {
t.Fatalf("media access=%#v err=%v", access, err)
}
if _, err := service.CreateMediaAccess(ctx, editor, segmentID, "DOWNLOAD", "未授权下载验证", "media-access-denied"); !errors.Is(err, ErrRecordingAccessDenied) {
t.Fatalf("download access error=%v", err)
}
revoked, err := service.RevokeRecordingAccess(ctx, owner, grant.ID, "media-revoke")
if err != nil || revoked.RevokedAt == nil {
t.Fatalf("revoked grant=%#v err=%v", revoked, err)
}
if _, err := service.GetConversationMedia(ctx, editor, created.ConversationID); !errors.Is(err, ErrRecordingAccessDenied) {
t.Fatalf("revoked conversation media error=%v", err)
}
if _, err := service.AcknowledgeLocalDelete(ctx, identity, segmentID, "DELETED", "", "media-delete-early"); !errors.Is(err, ErrLocalDeleteTooEarly) {
t.Fatalf("early local delete error=%v", err)
}
if _, err := pool.Exec(ctx, `UPDATE media_segment SET local_delete_after=now()-interval '1 second' WHERE public_id=$1::uuid`, segmentID); err != nil {
t.Fatal(err)
}
deleted, err := service.AcknowledgeLocalDelete(ctx, identity, segmentID, "DELETED", "", "media-delete")
if err != nil || deleted.LocalDeleteStatus != "DELETED" || deleted.LocalDeleteReportedAt == nil {
t.Fatalf("deleted segment=%#v err=%v", deleted, err)
}
}
func cleanupMediaFixture(t *testing.T, ctx context.Context, pool *pgxpool.Pool) {
t.Helper()
var ids []string
rows, _ := pool.Query(ctx, `SELECT id::text FROM conversation WHERE device_id=$1::uuid AND EXISTS(SELECT 1 FROM context_snapshot cs WHERE cs.id=conversation.context_snapshot_id AND cs.sanitized_summary LIKE '%media_integration%')`, mediaTestDeviceID)
if rows != nil {
for rows.Next() {
var id string
_ = rows.Scan(&id)
ids = append(ids, id)
}
rows.Close()
}
for _, id := range ids {
for _, statement := range []string{
`DELETE FROM outbox_event WHERE aggregate_id IN (SELECT public_id FROM upload_batch WHERE conversation_id=$1::uuid)`,
`DELETE FROM media_access_log WHERE conversation_id=$1::uuid`,
`DELETE FROM audit_event WHERE resource_id IN (SELECT public_id FROM upload_batch WHERE conversation_id=$1::uuid UNION SELECT public_id FROM media_segment WHERE conversation_id=$1::uuid)`,
`DELETE FROM upload_batch_segment WHERE upload_batch_id IN (SELECT id FROM upload_batch WHERE conversation_id=$1::uuid)`,
`DELETE FROM upload_batch WHERE conversation_id=$1::uuid`, `DELETE FROM media_segment WHERE conversation_id=$1::uuid`,
`DELETE FROM conversation_state_history WHERE conversation_id=$1::uuid`, `DELETE FROM rtc_task WHERE conversation_id=$1::uuid`, `DELETE FROM conversation WHERE id=$1::uuid`,
} {
_, _ = pool.Exec(ctx, statement, id)
}
}
_, _ = pool.Exec(ctx, `DELETE FROM audit_event WHERE request_id LIKE 'media-%'`)
_, _ = pool.Exec(ctx, `DELETE FROM recording_access_grant WHERE purpose='媒体集成测试播放'`)
_, _ = pool.Exec(ctx, `DELETE FROM context_snapshot WHERE sanitized_summary LIKE '%media_integration%'`)
_, _ = pool.Exec(ctx, `DELETE FROM consent_snapshot WHERE id IN ('90000000-0000-0000-0000-000000000803'::uuid) OR (subject_id='10000000-0000-0000-0000-000000000401'::uuid AND snapshot_version=9001)`)
_, _ = pool.Exec(ctx, `DELETE FROM consent_record WHERE id=$1::uuid`, mediaTestConsentID)
}
+53
View File
@@ -0,0 +1,53 @@
package media
import (
"strings"
"testing"
)
func TestNormalizeManifestIsDeterministic(t *testing.T) {
first := []SegmentDeclaration{
{Track: "far", SequenceNo: 2, Codec: "wav_pcm_s16le", ContentType: "audio/wav", DurationMS: 5000, ExpectedSize: 1200, ExpectedDigest: strings.Repeat("b", 64)},
{Track: "near", SequenceNo: 1, Codec: "wav_pcm_s16le", ContentType: "audio/wav", DurationMS: 5000, ExpectedSize: 2400, ExpectedDigest: strings.Repeat("a", 64)},
}
second := []SegmentDeclaration{first[1], first[0]}
digestA, normalized, total, err := normalizeManifest(first)
if err != nil {
t.Fatal(err)
}
digestB, _, _, err := normalizeManifest(second)
if err != nil {
t.Fatal(err)
}
if digestA != digestB {
t.Fatalf("manifest digest changed with input order: %s != %s", digestA, digestB)
}
if normalized[0].Track != "far" || normalized[1].Track != "near" {
t.Fatalf("manifest was not sorted: %#v", normalized)
}
if total != 3600 {
t.Fatalf("total = %d, want 3600", total)
}
}
func TestNormalizeManifestRejectsInvalidDeclarations(t *testing.T) {
base := SegmentDeclaration{Track: "near", SequenceNo: 1, Codec: "wav_pcm_s16le", ContentType: "audio/wav", DurationMS: 5000, ExpectedSize: 1200, ExpectedDigest: strings.Repeat("a", 64)}
tests := []struct {
name string
items []SegmentDeclaration
}{
{name: "empty"},
{name: "duplicate", items: []SegmentDeclaration{base, base}},
{name: "track", items: []SegmentDeclaration{func() SegmentDeclaration { item := base; item.Track = "mixed"; return item }()}},
{name: "codec content type mismatch", items: []SegmentDeclaration{func() SegmentDeclaration { item := base; item.ContentType = "audio/opus"; return item }()}},
{name: "digest", items: []SegmentDeclaration{func() SegmentDeclaration { item := base; item.ExpectedDigest = "no"; return item }()}},
{name: "size", items: []SegmentDeclaration{func() SegmentDeclaration { item := base; item.ExpectedSize = 0; return item }()}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if _, _, _, err := normalizeManifest(tt.items); err == nil {
t.Fatal("normalizeManifest accepted invalid input")
}
})
}
}
+47
View File
@@ -0,0 +1,47 @@
package media
import "encoding/binary"
func validateAudio(codec string, declaredDurationMS int, prefix []byte) string {
if codec != "wav_pcm_s16le" {
return "CODEC_VALIDATION_UNAVAILABLE"
}
if len(prefix) < 44 || string(prefix[:4]) != "RIFF" || string(prefix[8:12]) != "WAVE" {
return "INVALID_AUDIO_CONTAINER"
}
var byteRate, dataSize uint32
var audioFormat, bitsPerSample uint16
for offset := 12; offset+8 <= len(prefix); {
chunkID := string(prefix[offset : offset+4])
chunkSize := int(binary.LittleEndian.Uint32(prefix[offset+4 : offset+8]))
dataStart := offset + 8
if chunkID == "data" {
dataSize = uint32(chunkSize)
break
}
if chunkSize < 0 || dataStart+chunkSize > len(prefix) {
break
}
switch chunkID {
case "fmt ":
if chunkSize < 16 {
return "INVALID_AUDIO_CONTAINER"
}
audioFormat = binary.LittleEndian.Uint16(prefix[dataStart : dataStart+2])
byteRate = binary.LittleEndian.Uint32(prefix[dataStart+8 : dataStart+12])
bitsPerSample = binary.LittleEndian.Uint16(prefix[dataStart+14 : dataStart+16])
}
offset = dataStart + chunkSize
if offset%2 != 0 {
offset++
}
}
if audioFormat != 1 || bitsPerSample != 16 || byteRate == 0 || dataSize == 0 {
return "UNSUPPORTED_AUDIO_ENCODING"
}
actualDurationMS := int((uint64(dataSize) * 1000) / uint64(byteRate))
if actualDurationMS != declaredDurationMS {
return "DURATION_MISMATCH"
}
return ""
}
+38
View File
@@ -0,0 +1,38 @@
package media
import (
"encoding/binary"
"testing"
)
func TestValidateAudioWAVPCM16(t *testing.T) {
content := syntheticWAV(8000, 1, 800)
if code := validateAudio("wav_pcm_s16le", 100, content); code != "" {
t.Fatalf("valid WAV rejected with %s", code)
}
if code := validateAudio("wav_pcm_s16le", 101, content); code != "DURATION_MISMATCH" {
t.Fatalf("duration mismatch code = %s", code)
}
if code := validateAudio("opus", 100, content); code != "CODEC_VALIDATION_UNAVAILABLE" {
t.Fatalf("unsupported codec code = %s", code)
}
}
func syntheticWAV(sampleRate uint32, channels uint16, samples int) []byte {
dataSize := samples * int(channels) * 2
content := make([]byte, 44+dataSize)
copy(content[0:4], "RIFF")
binary.LittleEndian.PutUint32(content[4:8], uint32(len(content)-8))
copy(content[8:12], "WAVE")
copy(content[12:16], "fmt ")
binary.LittleEndian.PutUint32(content[16:20], 16)
binary.LittleEndian.PutUint16(content[20:22], 1)
binary.LittleEndian.PutUint16(content[22:24], channels)
binary.LittleEndian.PutUint32(content[24:28], sampleRate)
binary.LittleEndian.PutUint32(content[28:32], sampleRate*uint32(channels)*2)
binary.LittleEndian.PutUint16(content[32:34], channels*2)
binary.LittleEndian.PutUint16(content[34:36], 16)
copy(content[36:40], "data")
binary.LittleEndian.PutUint32(content[40:44], uint32(dataSize))
return content
}