chore: establish local platform baseline
This commit is contained in:
Codex Local
@@ -0,0 +1,229 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/conversation"
|
||||
"data-terminal-cloud/internal/deviceauth"
|
||||
"data-terminal-cloud/internal/rtc"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type HeartbeatResult struct {
|
||||
Response HeartbeatResponse
|
||||
InternalID string
|
||||
DevicePublicID string
|
||||
Status conversation.Status
|
||||
SequenceNo int64
|
||||
Updated bool
|
||||
Credential *CredentialRenewal
|
||||
}
|
||||
|
||||
type CredentialRenewal struct {
|
||||
RoomID, DeviceUserID, AgentUserID string
|
||||
IssuedAt, ExpiresAt time.Time
|
||||
}
|
||||
|
||||
type HeartbeatServiceOptions struct {
|
||||
RTCAppID string
|
||||
LeaseTTL time.Duration
|
||||
TokenTTL time.Duration
|
||||
CredentialRenewalWindow time.Duration
|
||||
}
|
||||
|
||||
type HeartbeatService struct {
|
||||
pool *pgxpool.Pool
|
||||
repository *Repository
|
||||
leases LeaseCache
|
||||
tokens rtc.TokenIssuer
|
||||
options HeartbeatServiceOptions
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewHeartbeatService(pool *pgxpool.Pool, repository *Repository, leases LeaseCache, tokens rtc.TokenIssuer, options HeartbeatServiceOptions) (*HeartbeatService, error) {
|
||||
if pool == nil || repository == nil || leases == nil || tokens == nil || options.RTCAppID == "" || options.LeaseTTL <= 0 || options.TokenTTL <= 0 || options.CredentialRenewalWindow <= 0 || options.CredentialRenewalWindow >= options.TokenTTL {
|
||||
return nil, errors.New("heartbeat database, repository, lease cache, token issuer and valid TTL options are required")
|
||||
}
|
||||
return &HeartbeatService{pool: pool, repository: repository, leases: leases, tokens: tokens, options: options, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (s *HeartbeatService) Heartbeat(ctx context.Context, identity deviceauth.Identity, conversationID string, input HeartbeatInput) (HeartbeatResponse, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return HeartbeatResponse{}, fmt.Errorf("begin heartbeat transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
now := s.now().UTC()
|
||||
result, err := s.repository.Heartbeat(ctx, tx, identity, conversationID, input, now, s.options.LeaseTTL, s.options.TokenTTL, s.options.CredentialRenewalWindow)
|
||||
if err != nil {
|
||||
return HeartbeatResponse{}, err
|
||||
}
|
||||
if result.Credential != nil {
|
||||
token, issueErr := s.tokens.Issue(result.Credential.RoomID, result.Credential.DeviceUserID, result.Credential.ExpiresAt)
|
||||
if issueErr != nil {
|
||||
return HeartbeatResponse{}, fmt.Errorf("renew RTC credential: %w", issueErr)
|
||||
}
|
||||
result.Response.Actions = append(result.Response.Actions, HeartbeatAction{Type: "renew_rtc_credential", RTC: &RTCGrant{
|
||||
AppID: s.options.RTCAppID, RoomID: result.Credential.RoomID, UserID: result.Credential.DeviceUserID,
|
||||
AgentUserID: result.Credential.AgentUserID, Token: token, ExpiresAt: result.Credential.ExpiresAt,
|
||||
}})
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return HeartbeatResponse{}, fmt.Errorf("commit heartbeat: %w", err)
|
||||
}
|
||||
if result.Updated && result.Response.LeaseExpiresAt != nil {
|
||||
// PostgreSQL is authoritative. A missing Redis lease is recoverable and must not
|
||||
// turn an already committed, sequence-protected heartbeat into an ambiguous retry.
|
||||
_ = s.leases.Refresh(ctx, LeaseRecord{
|
||||
ConversationID: conversationID, DeviceID: result.DevicePublicID,
|
||||
SequenceNo: result.SequenceNo, Status: string(result.Status),
|
||||
ExpiresAt: *result.Response.LeaseExpiresAt, ObservedAt: now,
|
||||
})
|
||||
}
|
||||
return result.Response, nil
|
||||
}
|
||||
|
||||
func (r *Repository) Heartbeat(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
identity deviceauth.Identity,
|
||||
conversationPublicID string,
|
||||
input HeartbeatInput,
|
||||
now time.Time,
|
||||
leaseTTL, tokenTTL, credentialRenewalWindow time.Duration,
|
||||
) (HeartbeatResult, error) {
|
||||
var internalID string
|
||||
var status conversation.Status
|
||||
var lastSequence, desiredRevision int64
|
||||
var currentLease, startedAt *time.Time
|
||||
var credentialIssuedAt, credentialExpiresAt, conversationCreatedAt time.Time
|
||||
var roomID, deviceRTCUserID, agentRTCUserID string
|
||||
var credentialRenewalSupported bool
|
||||
var maxDurationSec *int
|
||||
var inProgressAction *string
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT c.id::text, c.status, c.last_heartbeat_seq, c.lease_expires_at,
|
||||
dt.desired_revision,c.started_at,cub.max_call_duration_sec,cub.in_progress_action,
|
||||
r.credential_issued_at,r.credential_expires_at,c.created_at,r.room_id,r.device_rtc_user_id,r.agent_rtc_user_id,
|
||||
c.rtc_credential_renewal_supported
|
||||
FROM conversation c
|
||||
JOIN device_twin dt ON dt.device_id=c.device_id AND dt.tenant_id=c.tenant_id
|
||||
JOIN rtc_task r ON r.conversation_id=c.id
|
||||
LEFT JOIN conversation_usage_budget cub ON cub.conversation_id=c.id AND cub.tenant_id=c.tenant_id
|
||||
WHERE c.public_id=$1::uuid AND c.device_id=$2::uuid AND c.tenant_id=$3::uuid
|
||||
FOR UPDATE OF c, dt, r`, conversationPublicID, identity.ID, identity.TenantID,
|
||||
).Scan(&internalID, &status, &lastSequence, ¤tLease, &desiredRevision, &startedAt, &maxDurationSec, &inProgressAction,
|
||||
&credentialIssuedAt, &credentialExpiresAt, &conversationCreatedAt, &roomID, &deviceRTCUserID, &agentRTCUserID,
|
||||
&credentialRenewalSupported)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return HeartbeatResult{}, ErrConversationNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return HeartbeatResult{}, fmt.Errorf("read heartbeat state: %w", err)
|
||||
}
|
||||
result := HeartbeatResult{
|
||||
InternalID: internalID, DevicePublicID: identity.PublicID, Status: status,
|
||||
SequenceNo: input.SequenceNo,
|
||||
Response: HeartbeatResponse{
|
||||
LeaseExpiresAt: currentLease, Actions: heartbeatActions(status),
|
||||
DesiredConfigRevision: desiredRevision, Commands: []json.RawMessage{},
|
||||
},
|
||||
}
|
||||
if !input.RTC.Joined || conversation.IsTerminal(input.ConversationStatus) || input.ConversationStatus == conversation.StatusStopping {
|
||||
result.Response.Actions = []HeartbeatAction{{Type: "stop"}}
|
||||
}
|
||||
if startedAt != nil && maxDurationSec != nil && !now.Before(startedAt.Add(time.Duration(*maxDurationSec)*time.Second)) {
|
||||
action := "stop_after_turn"
|
||||
if inProgressAction != nil && *inProgressAction == "STOP_IMMEDIATELY" {
|
||||
action = "stop"
|
||||
}
|
||||
result.Response.Actions = []HeartbeatAction{{Type: action}}
|
||||
}
|
||||
if input.ReportedConfigRevision > desiredRevision {
|
||||
return HeartbeatResult{}, ErrConfigVersionConflict
|
||||
}
|
||||
canRenewCredential := credentialRenewalSupported && len(result.Response.Actions) == 1 && result.Response.Actions[0].Type == "continue"
|
||||
storedCredentialIsNewer := canRenewCredential && credentialIssuedAt.After(conversationCreatedAt) &&
|
||||
(input.RTCCredentialExpiresAt == nil || input.RTCCredentialExpiresAt.Before(credentialExpiresAt))
|
||||
if storedCredentialIsNewer {
|
||||
result.Credential = &CredentialRenewal{RoomID: roomID, DeviceUserID: deviceRTCUserID, AgentUserID: agentRTCUserID, IssuedAt: credentialIssuedAt, ExpiresAt: credentialExpiresAt}
|
||||
}
|
||||
if input.SequenceNo <= lastSequence || conversation.IsTerminal(status) {
|
||||
return result, nil
|
||||
}
|
||||
if status != conversation.StatusStartingAgent && status != conversation.StatusActive && status != conversation.StatusStopping {
|
||||
return HeartbeatResult{}, ErrConversationStateConflict
|
||||
}
|
||||
if canRenewCredential && result.Credential == nil && !now.Add(credentialRenewalWindow).Before(credentialExpiresAt) {
|
||||
credentialIssuedAt = now
|
||||
credentialExpiresAt = now.Add(tokenTTL)
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET credential_issued_at=$2,credential_expires_at=$3,updated_at=$2 WHERE conversation_id=$1::uuid`, internalID, credentialIssuedAt, credentialExpiresAt); err != nil {
|
||||
return HeartbeatResult{}, fmt.Errorf("rotate RTC credential metadata: %w", err)
|
||||
}
|
||||
result.Credential = &CredentialRenewal{RoomID: roomID, DeviceUserID: deviceRTCUserID, AgentUserID: agentRTCUserID, IssuedAt: credentialIssuedAt, ExpiresAt: credentialExpiresAt}
|
||||
}
|
||||
|
||||
reportedHeartbeat, err := json.Marshal(map[string]any{
|
||||
"sequence_no": input.SequenceNo, "device_time": input.DeviceTime,
|
||||
"monotonic_ms": input.MonotonicMS, "conversation_status": input.ConversationStatus,
|
||||
"rtc": input.RTC, "network": input.Network, "recording_queue": input.RecordingQueue,
|
||||
"last_error": input.LastError,
|
||||
})
|
||||
if err != nil {
|
||||
return HeartbeatResult{}, fmt.Errorf("encode reported heartbeat: %w", err)
|
||||
}
|
||||
if status == conversation.StatusStartingAgent || status == conversation.StatusActive {
|
||||
leaseExpiresAt := now.Add(leaseTTL)
|
||||
result.Response.LeaseExpiresAt = &leaseExpiresAt
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE conversation SET last_heartbeat_seq=$2, lease_expires_at=$3,
|
||||
version=version+1, updated_at=$4 WHERE id=$1::uuid`,
|
||||
internalID, input.SequenceNo, leaseExpiresAt, now)
|
||||
if err == nil {
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE rtc_task SET lease_expires_at=$2, last_checked_at=$3, updated_at=$3
|
||||
WHERE conversation_id=$1::uuid`, internalID, leaseExpiresAt, now)
|
||||
}
|
||||
} else {
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE conversation SET last_heartbeat_seq=$2, version=version+1, updated_at=$3
|
||||
WHERE id=$1::uuid`, internalID, input.SequenceNo, now)
|
||||
}
|
||||
if err != nil {
|
||||
return HeartbeatResult{}, fmt.Errorf("refresh conversation heartbeat: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE device_twin SET
|
||||
reported_revision=CASE
|
||||
WHEN reported_revision IS NULL OR reported_revision <= $2 THEN $2
|
||||
ELSE reported_revision END,
|
||||
reported_config=CASE
|
||||
WHEN reported_revision IS NULL OR reported_revision <= $2
|
||||
THEN reported_config || jsonb_build_object('heartbeat', $3::jsonb)
|
||||
ELSE reported_config END,
|
||||
firmware_version=CASE WHEN NULLIF($4,'') IS NULL THEN firmware_version ELSE $4 END,
|
||||
last_reported_at=$5, updated_at=$5
|
||||
WHERE device_id=$1::uuid AND tenant_id=$6::uuid`,
|
||||
identity.ID, input.ReportedConfigRevision, reportedHeartbeat, input.FirmwareVersion, now, identity.TenantID)
|
||||
if err != nil {
|
||||
return HeartbeatResult{}, fmt.Errorf("update device heartbeat report: %w", err)
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `SELECT record_device_presence($1::uuid,$2::uuid,$3)`, identity.TenantID, identity.ID, now); err != nil {
|
||||
return HeartbeatResult{}, fmt.Errorf("record device presence: %w", err)
|
||||
}
|
||||
result.Updated = true
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func heartbeatActions(status conversation.Status) []HeartbeatAction {
|
||||
if status == conversation.StatusStartingAgent || status == conversation.StatusActive {
|
||||
return []HeartbeatAction{{Type: "continue"}}
|
||||
}
|
||||
return []HeartbeatAction{{Type: "stop"}}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type LeaseRecord struct {
|
||||
ConversationID string
|
||||
DeviceID string
|
||||
SequenceNo int64
|
||||
Status string
|
||||
ExpiresAt time.Time
|
||||
ObservedAt time.Time
|
||||
}
|
||||
|
||||
type LeaseCache interface {
|
||||
Refresh(context.Context, LeaseRecord) error
|
||||
End(context.Context, string, string, time.Time) error
|
||||
}
|
||||
|
||||
type RedisLeaseCache struct {
|
||||
client *redis.Client
|
||||
prefix string
|
||||
}
|
||||
|
||||
var refreshLeaseScript = redis.NewScript(`
|
||||
local current = redis.call('HGET', KEYS[1], 'sequence_no')
|
||||
if current and tonumber(current) >= tonumber(ARGV[1]) then
|
||||
return 0
|
||||
end
|
||||
redis.call('HSET', KEYS[1],
|
||||
'sequence_no', ARGV[1], 'status', ARGV[2], 'device_id', ARGV[3],
|
||||
'lease_expires_at', ARGV[4], 'observed_at', ARGV[5])
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[6])
|
||||
redis.call('HSET', KEYS[2],
|
||||
'online', 'true', 'conversation_id', ARGV[7], 'sequence_no', ARGV[1],
|
||||
'last_seen_at', ARGV[5])
|
||||
redis.call('PEXPIRE', KEYS[2], ARGV[8])
|
||||
return 1
|
||||
`)
|
||||
|
||||
func NewRedisLeaseCache(client *redis.Client, prefix string) (*RedisLeaseCache, error) {
|
||||
if client == nil || prefix == "" {
|
||||
return nil, errors.New("redis client and lease key prefix are required")
|
||||
}
|
||||
return &RedisLeaseCache{client: client, prefix: prefix}, nil
|
||||
}
|
||||
|
||||
func (c *RedisLeaseCache) Refresh(ctx context.Context, record LeaseRecord) error {
|
||||
ttl := time.Until(record.ExpiresAt)
|
||||
if ttl <= 0 || record.SequenceNo < 1 || record.ConversationID == "" || record.DeviceID == "" {
|
||||
return errors.New("active lease record is invalid or already expired")
|
||||
}
|
||||
presenceTTL := ttl * 2
|
||||
_, err := refreshLeaseScript.Run(ctx, c.client, []string{
|
||||
c.prefix + ":conversation:lease:" + record.ConversationID,
|
||||
c.prefix + ":device:presence:" + record.DeviceID,
|
||||
},
|
||||
strconv.FormatInt(record.SequenceNo, 10), record.Status, record.DeviceID,
|
||||
record.ExpiresAt.UTC().Format(time.RFC3339Nano), record.ObservedAt.UTC().Format(time.RFC3339Nano),
|
||||
ttl.Milliseconds(), record.ConversationID, presenceTTL.Milliseconds(),
|
||||
).Result()
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *RedisLeaseCache) End(ctx context.Context, conversationID, deviceID string, observedAt time.Time) error {
|
||||
if conversationID == "" || deviceID == "" {
|
||||
return errors.New("conversation and device identifiers are required")
|
||||
}
|
||||
leaseKey := c.prefix + ":conversation:lease:" + conversationID
|
||||
presenceKey := c.prefix + ":device:presence:" + deviceID
|
||||
pipe := c.client.TxPipeline()
|
||||
pipe.Del(ctx, leaseKey)
|
||||
pipe.HSet(ctx, presenceKey, "conversation_id", "", "status", "idle", "last_seen_at", observedAt.UTC().Format(time.RFC3339Nano))
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/cache"
|
||||
)
|
||||
|
||||
func TestRedisLeaseCacheRejectsOutOfOrderRefresh(t *testing.T) {
|
||||
redisAddress := os.Getenv("TEST_REDIS_ADDR")
|
||||
if redisAddress == "" {
|
||||
t.Skip("TEST_REDIS_ADDR is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
client := cache.Open(redisAddress, "", 0)
|
||||
defer client.Close()
|
||||
prefix := "voice-platform-lease-test-" + time.Now().UTC().Format("150405.000000000")
|
||||
leases, err := NewRedisLeaseCache(client, prefix)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conversationID := "10000000-0000-0000-0000-000000009901"
|
||||
deviceID := "10000000-0000-0000-0000-000000009902"
|
||||
leaseKey := prefix + ":conversation:lease:" + conversationID
|
||||
presenceKey := prefix + ":device:presence:" + deviceID
|
||||
t.Cleanup(func() { client.Del(context.Background(), leaseKey, presenceKey) })
|
||||
|
||||
now := time.Now().UTC()
|
||||
if err := leases.Refresh(ctx, LeaseRecord{
|
||||
ConversationID: conversationID, DeviceID: deviceID, SequenceNo: 10,
|
||||
Status: "ACTIVE", ObservedAt: now, ExpiresAt: now.Add(time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := leases.Refresh(ctx, LeaseRecord{
|
||||
ConversationID: conversationID, DeviceID: deviceID, SequenceNo: 9,
|
||||
Status: "STOPPING", ObservedAt: now.Add(time.Second), ExpiresAt: now.Add(2 * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values, err := client.HMGet(ctx, leaseKey, "sequence_no", "status").Result()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if values[0] != "10" || values[1] != "ACTIVE" {
|
||||
t.Fatalf("out-of-order heartbeat changed lease: %#v", values)
|
||||
}
|
||||
if online, err := client.HGet(ctx, presenceKey, "online").Result(); err != nil || online != "true" {
|
||||
t.Fatalf("presence online=%q err=%v", online, err)
|
||||
}
|
||||
if err := leases.End(ctx, "", deviceID, now); err == nil {
|
||||
t.Fatal("empty conversation identifier was accepted")
|
||||
}
|
||||
if err := leases.End(ctx, conversationID, deviceID, now.Add(2*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exists, err := client.Exists(ctx, leaseKey).Result(); err != nil || exists != 0 {
|
||||
t.Fatalf("ended lease exists=%d err=%v", exists, err)
|
||||
}
|
||||
values, err = client.HMGet(ctx, presenceKey, "conversation_id", "status").Result()
|
||||
if err != nil || values[0] != "" || values[1] != "idle" {
|
||||
t.Fatalf("ended presence=%#v err=%v", values, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var ErrProviderUnavailable = errors.New("provider is temporarily unavailable")
|
||||
|
||||
type ProviderHealth struct {
|
||||
pool *pgxpool.Pool
|
||||
failureThreshold int
|
||||
cooldown time.Duration
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewProviderHealth(pool *pgxpool.Pool, failureThreshold int, cooldown time.Duration) (*ProviderHealth, error) {
|
||||
if pool == nil || failureThreshold < 1 || cooldown <= 0 {
|
||||
return nil, errors.New("provider health requires a database, positive failure threshold and cooldown")
|
||||
}
|
||||
return &ProviderHealth{pool: pool, failureThreshold: failureThreshold, cooldown: cooldown, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (h *ProviderHealth) AllowStart(ctx context.Context, tenantID, provider string) (bool, error) {
|
||||
now := h.now().UTC()
|
||||
var status string
|
||||
var openUntil *time.Time
|
||||
err := h.pool.QueryRow(ctx, `
|
||||
INSERT INTO provider_dependency_state(tenant_id,provider,status,updated_at)
|
||||
VALUES($1::uuid,$2,'AVAILABLE',$3)
|
||||
ON CONFLICT (tenant_id,provider) DO UPDATE SET provider=EXCLUDED.provider
|
||||
RETURNING status,open_until`, tenantID, provider, now).Scan(&status, &openUntil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return status != "UNAVAILABLE" || openUntil == nil || !now.Before(*openUntil), nil
|
||||
}
|
||||
|
||||
func (h *ProviderHealth) RecordFailure(ctx context.Context, tenantID, provider, errorCode string) error {
|
||||
now := h.now().UTC()
|
||||
code := strings.ToUpper(strings.TrimSpace(errorCode))
|
||||
if code == "" {
|
||||
code = "PROVIDER_TEMPORARY_ERROR"
|
||||
}
|
||||
_, err := h.pool.Exec(ctx, `
|
||||
INSERT INTO provider_dependency_state(
|
||||
tenant_id,provider,status,consecutive_failures,open_until,last_error_code,last_failure_at,updated_at
|
||||
) VALUES(
|
||||
$1::uuid,$2,CASE WHEN $3<=1 THEN 'UNAVAILABLE' ELSE 'AVAILABLE' END,1,
|
||||
CASE WHEN $3<=1 THEN $4::timestamptz ELSE NULL END,$5,$6,$6
|
||||
)
|
||||
ON CONFLICT (tenant_id,provider) DO UPDATE SET
|
||||
consecutive_failures=provider_dependency_state.consecutive_failures+1,
|
||||
status=CASE WHEN provider_dependency_state.consecutive_failures+1 >= $3 THEN 'UNAVAILABLE' ELSE provider_dependency_state.status END,
|
||||
open_until=CASE WHEN provider_dependency_state.consecutive_failures+1 >= $3 THEN $4::timestamptz ELSE provider_dependency_state.open_until END,
|
||||
last_error_code=$5,last_failure_at=$6,row_version=provider_dependency_state.row_version+1,updated_at=$6`,
|
||||
tenantID, provider, h.failureThreshold, now.Add(h.cooldown), code, now)
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *ProviderHealth) RecordSuccess(ctx context.Context, tenantID, provider string) error {
|
||||
now := h.now().UTC()
|
||||
_, err := h.pool.Exec(ctx, `
|
||||
INSERT INTO provider_dependency_state(tenant_id,provider,status,last_success_at,updated_at)
|
||||
VALUES($1::uuid,$2,'AVAILABLE',$3,$3)
|
||||
ON CONFLICT (tenant_id,provider) DO UPDATE SET
|
||||
status='AVAILABLE',consecutive_failures=0,open_until=NULL,last_error_code=NULL,
|
||||
last_success_at=$3,row_version=provider_dependency_state.row_version+1,updated_at=$3`, tenantID, provider, now)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/database"
|
||||
)
|
||||
|
||||
func TestProviderHealthPersistsCircuitAndRecovers(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "provider-health-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
tenantID := "10000000-0000-0000-0000-000000000001"
|
||||
provider := "provider-health-test"
|
||||
defer pool.Exec(context.Background(), `DELETE FROM provider_dependency_state WHERE tenant_id=$1::uuid AND provider=$2`, tenantID, provider)
|
||||
|
||||
health, err := NewProviderHealth(pool, 3, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)
|
||||
health.now = func() time.Time { return now }
|
||||
for attempt := 1; attempt <= 2; attempt++ {
|
||||
if err := health.RecordFailure(ctx, tenantID, provider, "PROVIDER_START_TEMPORARY"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allowed, err := health.AllowStart(ctx, tenantID, provider)
|
||||
if err != nil || !allowed {
|
||||
t.Fatalf("attempt %d allowed=%v err=%v", attempt, allowed, err)
|
||||
}
|
||||
}
|
||||
if err := health.RecordFailure(ctx, tenantID, provider, "PROVIDER_START_TEMPORARY"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allowed, err := health.AllowStart(ctx, tenantID, provider)
|
||||
if err != nil || allowed {
|
||||
t.Fatalf("open circuit allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
var status, errorCode string
|
||||
var failures int
|
||||
if err := pool.QueryRow(ctx, `SELECT status,consecutive_failures,last_error_code FROM provider_dependency_state WHERE tenant_id=$1::uuid AND provider=$2`, tenantID, provider).Scan(&status, &failures, &errorCode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "UNAVAILABLE" || failures != 3 || errorCode != "PROVIDER_START_TEMPORARY" {
|
||||
t.Fatalf("state=%s failures=%d error=%s", status, failures, errorCode)
|
||||
}
|
||||
now = now.Add(61 * time.Second)
|
||||
allowed, err = health.AllowStart(ctx, tenantID, provider)
|
||||
if err != nil || !allowed {
|
||||
t.Fatalf("cooldown probe allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
if err := health.RecordSuccess(ctx, tenantID, provider); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allowed, err = health.AllowStart(ctx, tenantID, provider)
|
||||
if err != nil || !allowed {
|
||||
t.Fatalf("recovered allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT status,consecutive_failures FROM provider_dependency_state WHERE tenant_id=$1::uuid AND provider=$2`, tenantID, provider).Scan(&status, &failures); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "AVAILABLE" || failures != 0 {
|
||||
t.Fatalf("recovered state=%s failures=%d", status, failures)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,754 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/billing"
|
||||
"data-terminal-cloud/internal/conversation"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
"data-terminal-cloud/internal/ratelimit"
|
||||
"data-terminal-cloud/internal/voiceprovider"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrProviderTaskNotFound = errors.New("provider task was not found")
|
||||
ErrProviderEventConflict = errors.New("provider event key was reused with different content")
|
||||
ErrProviderRateLimited = errors.New("provider start rate limit reached")
|
||||
ErrProviderCapacity = errors.New("provider active task capacity reached")
|
||||
)
|
||||
|
||||
type providerStartRateLimiter interface {
|
||||
Allow(context.Context, string, string) (ratelimit.Result, error)
|
||||
}
|
||||
|
||||
type StartProcessor struct {
|
||||
pool *pgxpool.Pool
|
||||
provider voiceprovider.Client
|
||||
health *ProviderHealth
|
||||
initialLeaseTTL time.Duration
|
||||
startLimiter providerStartRateLimiter
|
||||
maxActiveTasks int
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// ConfigureStartProtection enables the provider-wide RPS bucket and durable
|
||||
// active-task capacity reservation used by every worker replica.
|
||||
func (p *StartProcessor) ConfigureStartProtection(limiter providerStartRateLimiter, maxActiveTasks int) error {
|
||||
if limiter == nil || maxActiveTasks <= 0 {
|
||||
return errors.New("provider start limiter and positive active task capacity are required")
|
||||
}
|
||||
p.startLimiter = limiter
|
||||
p.maxActiveTasks = maxActiveTasks
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewStartProcessor(pool *pgxpool.Pool, provider voiceprovider.Client, health *ProviderHealth, initialLeaseTTL time.Duration) (*StartProcessor, error) {
|
||||
if pool == nil || provider == nil || health == nil || initialLeaseTTL <= 0 {
|
||||
return nil, errors.New("database pool, voice provider client, provider health and positive lease TTL are required")
|
||||
}
|
||||
return &StartProcessor{pool: pool, provider: provider, health: health, initialLeaseTTL: initialLeaseTTL, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (p *StartProcessor) Handle(ctx context.Context, event outbox.Event) error {
|
||||
var request voiceprovider.StartRequest
|
||||
var conversationStatus conversation.Status
|
||||
var taskStatus, tenantID, providerName string
|
||||
err := p.pool.QueryRow(ctx, `
|
||||
SELECT c.public_id::text, c.status, r.provider_task_ref, r.room_id,
|
||||
r.device_rtc_user_id, r.agent_rtc_user_id, r.status,cs.sanitized_summary,
|
||||
c.engine_mode,c.tenant_id::text,r.provider,cs.prompt_bundle_version,
|
||||
COALESCE(bundle.public_id::text,''),COALESCE(bundle.payload->>'persona_version_id',''),
|
||||
COALESCE(bundle.payload->>'voice_version_id',''),COALESCE(bundle.payload->>'model_version_id',''),
|
||||
COALESCE(bundle.payload->>'routing_version_id',''),COALESCE(persona.payload->>'system_prompt',''),
|
||||
COALESCE(persona.payload->>'tone',''),COALESCE(voice.payload->>'voice_resource_ref',''),
|
||||
COALESCE(model.payload->>'model_id','')
|
||||
FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id
|
||||
JOIN context_snapshot cs ON cs.id=c.context_snapshot_id AND cs.tenant_id=c.tenant_id
|
||||
LEFT JOIN runtime_config_version bundle
|
||||
ON bundle.tenant_id=c.tenant_id AND bundle.config_kind='AGENT_BUNDLE'
|
||||
AND cs.prompt_bundle_version='cfg:' || bundle.public_id::text
|
||||
LEFT JOIN runtime_config_version persona
|
||||
ON persona.tenant_id=bundle.tenant_id AND persona.config_kind='PERSONA'
|
||||
AND persona.public_id::text=bundle.payload->>'persona_version_id'
|
||||
LEFT JOIN runtime_config_version voice
|
||||
ON voice.tenant_id=bundle.tenant_id AND voice.config_kind='VOICE'
|
||||
AND voice.public_id::text=bundle.payload->>'voice_version_id'
|
||||
LEFT JOIN runtime_config_version model
|
||||
ON model.tenant_id=bundle.tenant_id AND model.config_kind='MODEL'
|
||||
AND model.public_id::text=bundle.payload->>'model_version_id'
|
||||
WHERE c.id=$1::uuid`, event.AggregateID,
|
||||
).Scan(&request.ConversationID, &conversationStatus, &request.TaskRef, &request.RoomID,
|
||||
&request.DeviceUserID, &request.AgentUserID, &taskStatus, &request.ContextSummary, &request.EngineMode, &tenantID, &providerName,
|
||||
&request.PromptBundleVersion, &request.AgentBundleVersion, &request.PersonaVersion, &request.VoiceVersion,
|
||||
&request.ModelVersion, &request.RoutingVersion, &request.SystemPrompt, &request.PersonaTone,
|
||||
&request.VoiceResourceRef, &request.ModelResourceRef)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrProviderTaskNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read provider start task: %w", err)
|
||||
}
|
||||
if conversationStatus != conversation.StatusStartingAgent {
|
||||
return nil
|
||||
}
|
||||
if taskStatus != "STARTING" && taskStatus != "UNKNOWN" {
|
||||
return fmt.Errorf("provider task status %s cannot be started", taskStatus)
|
||||
}
|
||||
allowed, err := p.health.AllowStart(ctx, tenantID, providerName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read provider health: %w", err)
|
||||
}
|
||||
if !allowed {
|
||||
return outbox.Retryable(ErrProviderUnavailable)
|
||||
}
|
||||
var response voiceprovider.StartResponse
|
||||
if taskStatus == "UNKNOWN" {
|
||||
queryResult, queryErr := p.provider.Query(ctx, voiceprovider.QueryRequest{
|
||||
TaskRef: request.TaskRef, ConversationID: request.ConversationID,
|
||||
})
|
||||
switch {
|
||||
case queryErr == nil && queryResult.Status == "ACTIVE":
|
||||
response = voiceprovider.StartResponse{TaskRef: queryResult.TaskRef, Status: queryResult.Status}
|
||||
case queryErr == nil && queryResult.Status == "STARTING":
|
||||
_, _ = p.pool.Exec(ctx, `UPDATE rtc_task SET status='STARTING',last_checked_at=$2,updated_at=$2 WHERE conversation_id=$1::uuid`, event.AggregateID, p.now().UTC())
|
||||
return outbox.Retryable(errors.New("provider start is still pending confirmation"))
|
||||
case queryErr == nil:
|
||||
return fmt.Errorf("provider query returned unsupported start reconciliation status %s", queryResult.Status)
|
||||
case !voiceprovider.IsNotFound(queryErr):
|
||||
if voiceprovider.IsTemporary(queryErr) {
|
||||
return outbox.Retryable(fmt.Errorf("query unknown provider start before retry: %w", queryErr))
|
||||
}
|
||||
return fmt.Errorf("query unknown provider start before retry: %w", queryErr)
|
||||
}
|
||||
// A provider-confirmed 404 is the only UNKNOWN case in which Start may
|
||||
// safely be sent again. The same stable task reference is retained.
|
||||
}
|
||||
newReservation := false
|
||||
if response.Status == "" && p.maxActiveTasks > 0 {
|
||||
newReservation, err = p.reserveProviderCapacity(ctx, event.AggregateID, providerName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if response.Status == "" && p.startLimiter != nil {
|
||||
limitResult, limitErr := p.startLimiter.Allow(ctx, "provider_start", providerName)
|
||||
if limitErr != nil {
|
||||
return fmt.Errorf("apply provider start rate limit: %w", limitErr)
|
||||
}
|
||||
if !limitResult.Allowed {
|
||||
return outbox.Retryable(fmt.Errorf("%w: retry after %s", ErrProviderRateLimited, limitResult.RetryAfter))
|
||||
}
|
||||
}
|
||||
if response.Status == "" && !newReservation {
|
||||
_, err = p.pool.Exec(ctx, `
|
||||
UPDATE rtc_task SET start_attempts=start_attempts+1, last_checked_at=$2, updated_at=$2
|
||||
WHERE conversation_id=$1::uuid`, event.AggregateID, p.now().UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("record provider start attempt: %w", err)
|
||||
}
|
||||
}
|
||||
if response.Status == "" {
|
||||
response, err = p.provider.Start(ctx, request)
|
||||
if err != nil {
|
||||
if voiceprovider.IsTemporary(err) {
|
||||
_, _ = p.pool.Exec(ctx, `UPDATE rtc_task SET status='UNKNOWN', updated_at=$2 WHERE conversation_id=$1::uuid AND status IN ('STARTING','UNKNOWN')`, event.AggregateID, p.now().UTC())
|
||||
if healthErr := p.health.RecordFailure(ctx, tenantID, providerName, "PROVIDER_START_TEMPORARY"); healthErr != nil {
|
||||
return fmt.Errorf("record provider start failure: %v; provider error: %w", healthErr, err)
|
||||
}
|
||||
return outbox.Retryable(err)
|
||||
}
|
||||
return p.failRejectedStart(ctx, event.AggregateID, tenantID, "provider_start_rejected", "provider permanently rejected voice chat start")
|
||||
}
|
||||
}
|
||||
if err := p.health.RecordSuccess(ctx, tenantID, providerName); err != nil {
|
||||
return fmt.Errorf("record provider start success: %w", err)
|
||||
}
|
||||
if response.Status != "STARTING" && response.Status != "ACTIVE" {
|
||||
return p.failRejectedStart(ctx, event.AggregateID, tenantID, "provider_start_invalid_response", "provider returned an unsupported voice chat start status")
|
||||
}
|
||||
now := p.now().UTC()
|
||||
if response.Status == "STARTING" {
|
||||
_, err = p.pool.Exec(ctx, `
|
||||
UPDATE rtc_task SET status=CASE WHEN status='ACTIVE' THEN 'ACTIVE' ELSE 'STARTING' END,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, event.AggregateID, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record provider start response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
leaseExpiresAt := now.Add(p.initialLeaseTTL)
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='ACTIVE', lease_expires_at=$2,
|
||||
last_checked_at=$3, updated_at=$3 WHERE conversation_id=$1::uuid`, event.AggregateID, leaseExpiresAt, now); err != nil {
|
||||
return fmt.Errorf("record provider active response: %w", err)
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE conversation SET status='ACTIVE', started_at=COALESCE(started_at,$2),
|
||||
lease_expires_at=$3, version=version+1, updated_at=$2
|
||||
WHERE id=$1::uuid AND status='STARTING_AGENT'`, event.AggregateID, now, leaseExpiresAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile active conversation: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 1 {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source
|
||||
) VALUES ($1::uuid, 'STARTING_AGENT', 'ACTIVE', 'provider idempotent start response confirmed active task', 'worker')`, event.AggregateID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (p *StartProcessor) failRejectedStart(ctx context.Context, conversationID, tenantID, failureCode, reason string) error {
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
now := p.now().UTC()
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE conversation SET status='FAILED',ended_at=$2,lease_expires_at=NULL,
|
||||
failure_code=$3,version=version+1,updated_at=$2
|
||||
WHERE id=$1::uuid AND status='STARTING_AGENT'`, conversationID, now, failureCode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fail permanently rejected provider start: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE rtc_task SET status='FAILED',lease_expires_at=NULL,
|
||||
last_checked_at=$2,updated_at=$2 WHERE conversation_id=$1::uuid`, conversationID, now); err != nil {
|
||||
return fmt.Errorf("fail rejected provider RTC task: %w", err)
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id,from_status,to_status,reason,source
|
||||
) VALUES ($1::uuid,'STARTING_AGENT','FAILED',$2,'worker')`, conversationID, reason); err != nil {
|
||||
return fmt.Errorf("record rejected provider start state: %w", err)
|
||||
}
|
||||
detail, _ := json.Marshal(map[string]string{"failure_code": failureCode})
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO audit_event (
|
||||
tenant_id,actor_type,action,resource_type,resource_id,purpose,result,detail
|
||||
) VALUES ($1::uuid,'system','conversation.agent_start_failed','conversation',$2::uuid,
|
||||
'provider_start','failure',$3::jsonb)`, tenantID, conversationID, detail); err != nil {
|
||||
return fmt.Errorf("audit rejected provider start: %w", err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (p *StartProcessor) reserveProviderCapacity(ctx context.Context, conversationID, providerName string) (bool, error) {
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, "provider-capacity:"+providerName); err != nil {
|
||||
return false, fmt.Errorf("lock provider capacity: %w", err)
|
||||
}
|
||||
var attempts int
|
||||
if err = tx.QueryRow(ctx, `SELECT start_attempts FROM rtc_task WHERE conversation_id=$1::uuid FOR UPDATE`, conversationID).Scan(&attempts); err != nil {
|
||||
return false, fmt.Errorf("read provider capacity reservation: %w", err)
|
||||
}
|
||||
if attempts > 0 {
|
||||
return false, tx.Commit(ctx)
|
||||
}
|
||||
var active int
|
||||
if err = tx.QueryRow(ctx, `
|
||||
SELECT count(*) FROM rtc_task
|
||||
WHERE provider=$1 AND (
|
||||
status IN ('ACTIVE','STOPPING') OR
|
||||
(status IN ('STARTING','UNKNOWN') AND start_attempts>0)
|
||||
)`, providerName).Scan(&active); err != nil {
|
||||
return false, fmt.Errorf("count provider capacity: %w", err)
|
||||
}
|
||||
if active >= p.maxActiveTasks {
|
||||
return false, outbox.Retryable(fmt.Errorf("%w: provider=%s active=%d maximum=%d", ErrProviderCapacity, providerName, active, p.maxActiveTasks))
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE rtc_task SET start_attempts=1,last_checked_at=$2,updated_at=$2 WHERE conversation_id=$1::uuid`, conversationID, p.now().UTC()); err != nil {
|
||||
return false, fmt.Errorf("reserve provider capacity: %w", err)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type StopProcessor struct {
|
||||
pool *pgxpool.Pool
|
||||
provider voiceprovider.Client
|
||||
health *ProviderHealth
|
||||
outbox *outbox.Store
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewStopProcessor(pool *pgxpool.Pool, provider voiceprovider.Client, health *ProviderHealth) (*StopProcessor, error) {
|
||||
if pool == nil || provider == nil || health == nil {
|
||||
return nil, errors.New("database pool, voice provider client and provider health are required")
|
||||
}
|
||||
return &StopProcessor{pool: pool, provider: provider, health: health, outbox: outbox.NewStore(pool), now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (p *StopProcessor) Handle(ctx context.Context, event outbox.Event) error {
|
||||
var request voiceprovider.StopRequest
|
||||
var conversationStatus conversation.Status
|
||||
var taskStatus, tenantID, providerName string
|
||||
err := p.pool.QueryRow(ctx, `
|
||||
SELECT c.public_id::text, c.status, COALESCE(c.stop_reason,'provider_failure'), r.provider_task_ref, r.room_id, r.status, c.tenant_id::text, r.provider
|
||||
FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id
|
||||
WHERE c.id=$1::uuid`, event.AggregateID,
|
||||
).Scan(&request.ConversationID, &conversationStatus, &request.Reason, &request.TaskRef, &request.RoomID, &taskStatus, &tenantID, &providerName)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrProviderTaskNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read provider stop task: %w", err)
|
||||
}
|
||||
if conversationStatus != conversation.StatusStopping || taskStatus == "COMPLETED" {
|
||||
return nil
|
||||
}
|
||||
if taskStatus != "STOPPING" && taskStatus != "UNKNOWN" && taskStatus != "FAILED" {
|
||||
return fmt.Errorf("provider task status %s cannot be stopped", taskStatus)
|
||||
}
|
||||
now := p.now().UTC()
|
||||
_, err = p.pool.Exec(ctx, `
|
||||
UPDATE rtc_task SET stop_attempts=stop_attempts+1, last_checked_at=$2, updated_at=$2
|
||||
WHERE conversation_id=$1::uuid`, event.AggregateID, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record provider stop attempt: %w", err)
|
||||
}
|
||||
response, err := p.provider.Stop(ctx, request)
|
||||
if err != nil {
|
||||
_, _ = p.pool.Exec(ctx, `UPDATE rtc_task SET status='UNKNOWN', updated_at=$2
|
||||
WHERE conversation_id=$1::uuid AND status IN ('STOPPING','UNKNOWN','FAILED')`, event.AggregateID, p.now().UTC())
|
||||
if voiceprovider.IsTemporary(err) {
|
||||
if healthErr := p.health.RecordFailure(ctx, tenantID, providerName, "PROVIDER_STOP_TEMPORARY"); healthErr != nil {
|
||||
return fmt.Errorf("record provider stop failure: %v; provider error: %w", healthErr, err)
|
||||
}
|
||||
}
|
||||
if voiceprovider.IsTemporary(err) {
|
||||
return outbox.Retryable(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := p.health.RecordSuccess(ctx, tenantID, providerName); err != nil {
|
||||
return fmt.Errorf("record provider stop success: %w", err)
|
||||
}
|
||||
if response.Status != "STOPPING" && response.Status != "COMPLETED" {
|
||||
return fmt.Errorf("provider returned unsupported stop status %s", response.Status)
|
||||
}
|
||||
now = p.now().UTC()
|
||||
if response.Status == "STOPPING" {
|
||||
_, err = p.pool.Exec(ctx, `UPDATE rtc_task SET status=CASE WHEN status='COMPLETED' THEN 'COMPLETED' ELSE 'STOPPING' END,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, event.AggregateID, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record provider stop response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED', lease_expires_at=NULL,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, event.AggregateID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE conversation SET status='COMPLETED', ended_at=COALESCE(ended_at,$2),
|
||||
lease_expires_at=NULL, version=version+1, updated_at=$2 WHERE id=$1::uuid AND status='STOPPING'`, event.AggregateID, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 1 {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source
|
||||
) VALUES ($1::uuid, 'STOPPING', 'COMPLETED', 'provider idempotent stop response confirmed completed task', 'worker')`, event.AggregateID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
deduplicationKey := "content-extraction-v1:" + event.AggregateID
|
||||
if _, err := p.outbox.Enqueue(ctx, tx, outbox.NewEvent{
|
||||
TenantID: &tenantID, AggregateType: "conversation", AggregateID: event.AggregateID,
|
||||
EventType: "conversation.content_extraction_requested", DeduplicationKey: &deduplicationKey,
|
||||
Payload: json.RawMessage(`{"generation_version":1}`),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("enqueue conversation content extraction: %w", err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
type ProviderEventResult struct {
|
||||
ConversationID string `json:"conversation_id"`
|
||||
Status conversation.Status `json:"status"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
Ignored bool `json:"ignored"`
|
||||
TurnsAccepted int `json:"turns_accepted,omitempty"`
|
||||
UsageAccepted int `json:"usage_accepted,omitempty"`
|
||||
}
|
||||
|
||||
type ProviderEventProcessor struct {
|
||||
pool *pgxpool.Pool
|
||||
outbox *outbox.Store
|
||||
initialLeaseTTL time.Duration
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewProviderEventProcessor(pool *pgxpool.Pool, initialLeaseTTL time.Duration) (*ProviderEventProcessor, error) {
|
||||
if pool == nil || initialLeaseTTL <= 0 {
|
||||
return nil, errors.New("database pool and positive initial lease TTL are required")
|
||||
}
|
||||
return &ProviderEventProcessor{pool: pool, outbox: outbox.NewStore(pool), initialLeaseTTL: initialLeaseTTL, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (p *ProviderEventProcessor) Process(ctx context.Context, event voiceprovider.CallbackEvent, rawBody []byte) (ProviderEventResult, error) {
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var internalID, tenantID, subjectID, publicID, taskProvider, retentionMode string
|
||||
var currentStatus conversation.Status
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT c.id::text, c.tenant_id::text, c.subject_id::text, c.public_id::text, c.status, r.provider, c.retention_mode
|
||||
FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id
|
||||
WHERE r.provider_task_ref=$1
|
||||
FOR UPDATE OF c, r`, event.TaskRef,
|
||||
).Scan(&internalID, &tenantID, &subjectID, &publicID, ¤tStatus, &taskProvider, &retentionMode)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ProviderEventResult{}, ErrProviderTaskNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("read callback task: %w", err)
|
||||
}
|
||||
if event.Provider != taskProvider || event.ConversationID != publicID {
|
||||
return ProviderEventResult{}, ErrProviderTaskNotFound
|
||||
}
|
||||
digest := sha256.Sum256(rawBody)
|
||||
digestText := hex.EncodeToString(digest[:])
|
||||
eventID := uuid.NewString()
|
||||
sanitized, _ := json.Marshal(map[string]any{
|
||||
"event_type": event.EventType, "status": event.Status, "occurred_at": event.OccurredAt, "turn_count": len(event.Turns), "usage_count": len(event.Usage),
|
||||
})
|
||||
tag, err := tx.Exec(ctx, `
|
||||
INSERT INTO provider_event (
|
||||
id, tenant_id, conversation_id, provider, provider_event_key,
|
||||
provider_task_ref, event_type, occurred_at, payload_digest, sanitized_payload
|
||||
) VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10::jsonb)
|
||||
ON CONFLICT (provider, provider_event_key) DO NOTHING`,
|
||||
eventID, tenantID, internalID, event.Provider, event.EventID, event.TaskRef,
|
||||
event.EventType, event.OccurredAt, digestText, sanitized,
|
||||
)
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("store provider event: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
var storedDigest string
|
||||
if err := tx.QueryRow(ctx, `SELECT payload_digest FROM provider_event WHERE provider=$1 AND provider_event_key=$2`, event.Provider, event.EventID).Scan(&storedDigest); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if storedDigest != digestText {
|
||||
return ProviderEventResult{}, ErrProviderEventConflict
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return ProviderEventResult{ConversationID: publicID, Status: currentStatus, Duplicate: true}, nil
|
||||
}
|
||||
|
||||
result := ProviderEventResult{ConversationID: publicID, Status: currentStatus}
|
||||
if event.EventType == "voice_chat.usage" && event.Status == "FINAL" {
|
||||
metrics := make([]billing.UsageMetricInput, 0, len(event.Usage))
|
||||
for _, metric := range event.Usage {
|
||||
metrics = append(metrics, billing.UsageMetricInput{UsageCategory: metric.UsageCategory, MetricCode: metric.MetricCode, Quantity: metric.Quantity, Unit: metric.Unit})
|
||||
}
|
||||
recorded, usageErr := billing.RecordProviderUsageTx(ctx, tx, tenantID, internalID, eventID, event.Provider, event.EventID, event.OccurredAt, metrics)
|
||||
if errors.Is(usageErr, billing.ErrUsageConflict) {
|
||||
return ProviderEventResult{}, ErrProviderEventConflict
|
||||
}
|
||||
if usageErr != nil {
|
||||
return ProviderEventResult{}, usageErr
|
||||
}
|
||||
if _, usageErr = tx.Exec(ctx, `UPDATE provider_event SET processing_status='processed',processed_at=$2 WHERE id=$1::uuid`, eventID, p.now().UTC()); usageErr != nil {
|
||||
return ProviderEventResult{}, usageErr
|
||||
}
|
||||
if usageErr = tx.Commit(ctx); usageErr != nil {
|
||||
return ProviderEventResult{}, usageErr
|
||||
}
|
||||
result.UsageAccepted = len(recorded)
|
||||
return result, nil
|
||||
}
|
||||
if event.EventType == "voice_chat.transcript.final" && event.Status == "FINAL" {
|
||||
return p.processTranscriptEvent(ctx, tx, eventID, internalID, tenantID, subjectID, retentionMode, event, result)
|
||||
}
|
||||
if event.EventType == "voice_chat.stopped" && event.Status == "COMPLETED" {
|
||||
return p.completeStoppedEvent(ctx, tx, eventID, internalID, tenantID, currentStatus, result)
|
||||
}
|
||||
if event.EventType == "voice_chat.failed" && event.Status == "FAILED" {
|
||||
return p.processFailedEvent(ctx, tx, eventID, internalID, currentStatus, result)
|
||||
}
|
||||
if event.EventType != "voice_chat.started" || event.Status != "ACTIVE" || currentStatus != conversation.StatusStartingAgent {
|
||||
result.Ignored = true
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status='ignored', processed_at=$2 WHERE id=$1::uuid`, eventID, p.now().UTC()); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
if err := conversation.ValidateTransition(currentStatus, conversation.StatusActive); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
now := p.now().UTC()
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE conversation SET status='ACTIVE', started_at=COALESCE(started_at,$2),
|
||||
lease_expires_at=$3, version=version+1, updated_at=$2
|
||||
WHERE id=$1::uuid AND status='STARTING_AGENT'`, internalID, now, now.Add(p.initialLeaseTTL))
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("activate conversation from provider event: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE rtc_task SET status='ACTIVE', lease_expires_at=$2, last_checked_at=$3, updated_at=$3
|
||||
WHERE conversation_id=$1::uuid`, internalID, now.Add(p.initialLeaseTTL), now)
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("activate RTC task: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, provider_event_id
|
||||
) VALUES ($1::uuid, 'STARTING_AGENT', 'ACTIVE', 'provider confirmed voice chat start', 'provider', $2::uuid)`, internalID, eventID)
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("record provider active state: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE provider_event SET processing_status='processed', processed_at=$2 WHERE id=$1::uuid`, eventID, now)
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
result.Status = conversation.StatusActive
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *ProviderEventProcessor) processTranscriptEvent(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
eventID, conversationID, tenantID, subjectID, retentionMode string,
|
||||
event voiceprovider.CallbackEvent,
|
||||
result ProviderEventResult,
|
||||
) (ProviderEventResult, error) {
|
||||
now := p.now().UTC()
|
||||
if retentionMode == "none" {
|
||||
result.Ignored = true
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status='ignored', processed_at=$2 WHERE id=$1::uuid`, eventID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
for _, turn := range event.Turns {
|
||||
providerRef := turn.ProviderRef
|
||||
if providerRef == "" {
|
||||
providerRef = turn.TurnKey
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
INSERT INTO conversation_turn (
|
||||
tenant_id,subject_id,conversation_id,turn_key,sequence_no,speaker,start_at,end_at,
|
||||
final_text,interrupted,provider_ref,source_type
|
||||
) VALUES ($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,$8,$9,$10,$11,'provider')
|
||||
ON CONFLICT (conversation_id,turn_key) DO NOTHING`, tenantID, subjectID, conversationID,
|
||||
turn.TurnKey, turn.SequenceNo, turn.Speaker, turn.StartAt, turn.EndAt, turn.FinalText, turn.Interrupted, providerRef)
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("store final transcript turn: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
var sequenceNo int64
|
||||
var speaker, finalText string
|
||||
var startAt, endAt time.Time
|
||||
var interrupted bool
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT sequence_no,speaker,start_at,end_at,final_text,interrupted
|
||||
FROM conversation_turn WHERE conversation_id=$1::uuid AND turn_key=$2`, conversationID, turn.TurnKey).Scan(
|
||||
&sequenceNo, &speaker, &startAt, &endAt, &finalText, &interrupted); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if sequenceNo != turn.SequenceNo || speaker != turn.Speaker || !startAt.Equal(turn.StartAt) ||
|
||||
!endAt.Equal(turn.EndAt) || finalText != turn.FinalText || interrupted != turn.Interrupted {
|
||||
return ProviderEventResult{}, ErrProviderEventConflict
|
||||
}
|
||||
continue
|
||||
}
|
||||
result.TurnsAccepted++
|
||||
}
|
||||
// A provider may deliver its final transcript after the stopped callback.
|
||||
// In that ordering the original extraction job may already have run before
|
||||
// any turns existed, so accepted late turns must create a new idempotent
|
||||
// generation request rather than leaving the conversation without content.
|
||||
if result.TurnsAccepted > 0 && conversation.IsTerminal(result.Status) {
|
||||
var generationVersion int64
|
||||
if err := tx.QueryRow(ctx, `SELECT COALESCE(max(sequence_no),1) FROM conversation_turn WHERE conversation_id=$1::uuid`, conversationID).Scan(&generationVersion); err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("read late transcript generation: %w", err)
|
||||
}
|
||||
digest := sha256.Sum256([]byte(event.Provider + ":" + event.EventID))
|
||||
deduplicationKey := "content-extraction-v1:" + conversationID + ":provider:" + hex.EncodeToString(digest[:16])
|
||||
payload, err := json.Marshal(map[string]int64{"generation_version": generationVersion})
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := p.outbox.Enqueue(ctx, tx, outbox.NewEvent{
|
||||
TenantID: &tenantID, AggregateType: "conversation", AggregateID: conversationID,
|
||||
EventType: "conversation.content_extraction_requested", DeduplicationKey: &deduplicationKey,
|
||||
Payload: payload,
|
||||
}); err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("enqueue late provider transcript extraction: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status='processed', processed_at=$2 WHERE id=$1::uuid`, eventID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *ProviderEventProcessor) completeStoppedEvent(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
eventID, internalID, tenantID string,
|
||||
currentStatus conversation.Status,
|
||||
result ProviderEventResult,
|
||||
) (ProviderEventResult, error) {
|
||||
if conversation.IsTerminal(currentStatus) || currentStatus == conversation.StatusRoomIssued || currentStatus == conversation.StatusPreparing {
|
||||
result.Ignored = true
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status='ignored', processed_at=$2 WHERE id=$1::uuid`, eventID, p.now().UTC()); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
now := p.now().UTC()
|
||||
if currentStatus == conversation.StatusStartingAgent || currentStatus == conversation.StatusActive {
|
||||
if err := conversation.ValidateTransition(currentStatus, conversation.StatusStopping); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET status='STOPPING', lease_expires_at=NULL,
|
||||
version=version+1, updated_at=$2 WHERE id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, provider_event_id
|
||||
) VALUES ($1::uuid, $2, 'STOPPING', 'provider ended task before platform stop completed', 'provider', $3::uuid)`,
|
||||
internalID, currentStatus, eventID); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET status='COMPLETED', ended_at=COALESCE(ended_at,$2),
|
||||
lease_expires_at=NULL, stop_reason=COALESCE(stop_reason,'provider_ended'),
|
||||
version=version+1, updated_at=$2 WHERE id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED', lease_expires_at=NULL,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, provider_event_id
|
||||
) VALUES ($1::uuid, 'STOPPING', 'COMPLETED', 'provider confirmed voice chat stop', 'provider', $2::uuid)`,
|
||||
internalID, eventID); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status='processed', processed_at=$2 WHERE id=$1::uuid`, eventID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
deduplicationKey := "content-extraction-v1:" + internalID
|
||||
if _, err := p.outbox.Enqueue(ctx, tx, outbox.NewEvent{
|
||||
TenantID: &tenantID, AggregateType: "conversation", AggregateID: internalID,
|
||||
EventType: "conversation.content_extraction_requested", DeduplicationKey: &deduplicationKey,
|
||||
Payload: json.RawMessage(`{"generation_version":1}`),
|
||||
}); err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("enqueue conversation content extraction: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
result.Status = conversation.StatusCompleted
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *ProviderEventProcessor) processFailedEvent(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
eventID, internalID string,
|
||||
currentStatus conversation.Status,
|
||||
result ProviderEventResult,
|
||||
) (ProviderEventResult, error) {
|
||||
now := p.now().UTC()
|
||||
switch currentStatus {
|
||||
case conversation.StatusStartingAgent:
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET status='FAILED', ended_at=$2,
|
||||
lease_expires_at=NULL, failure_code='provider_start_failed', version=version+1, updated_at=$2
|
||||
WHERE id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, provider_event_id
|
||||
) VALUES ($1::uuid, 'STARTING_AGENT', 'FAILED', 'provider reported voice chat failure', 'provider', $2::uuid)`, internalID, eventID); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
result.Status = conversation.StatusFailed
|
||||
case conversation.StatusActive:
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET status='STOPPING', lease_expires_at=NULL,
|
||||
failure_code='provider_runtime_failed', version=version+1, updated_at=$2 WHERE id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, provider_event_id
|
||||
) VALUES ($1::uuid, 'ACTIVE', 'STOPPING', 'provider reported runtime failure', 'provider', $2::uuid)`, internalID, eventID); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
result.Status = conversation.StatusStopping
|
||||
case conversation.StatusStopping:
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET failure_code=COALESCE(failure_code,'provider_stop_failed'), updated_at=$2 WHERE id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
default:
|
||||
result.Ignored = true
|
||||
}
|
||||
if !result.Ignored {
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='FAILED', lease_expires_at=NULL,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
}
|
||||
processingStatus := "processed"
|
||||
if result.Ignored {
|
||||
processingStatus = "ignored"
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status=$2, processed_at=$3 WHERE id=$1::uuid`, eventID, processingStatus, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/conversation"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type RecoveryOptions struct {
|
||||
ScanInterval time.Duration
|
||||
StartingAfter time.Duration
|
||||
StoppingAfter time.Duration
|
||||
BatchSize int
|
||||
}
|
||||
|
||||
type RecoveryScanner struct {
|
||||
pool *pgxpool.Pool
|
||||
outbox *outbox.Store
|
||||
logger *slog.Logger
|
||||
options RecoveryOptions
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type recoveryCandidate struct {
|
||||
InternalID string
|
||||
PublicID string
|
||||
TenantID string
|
||||
Status conversation.Status
|
||||
ProviderTaskRef string
|
||||
StopReason string
|
||||
Trigger string
|
||||
}
|
||||
|
||||
func NewRecoveryScanner(pool *pgxpool.Pool, outboxStore *outbox.Store, logger *slog.Logger, options RecoveryOptions) (*RecoveryScanner, error) {
|
||||
if pool == nil || outboxStore == nil || logger == nil || options.ScanInterval <= 0 ||
|
||||
options.StartingAfter <= 0 || options.StoppingAfter <= 0 || options.BatchSize < 1 {
|
||||
return nil, errors.New("valid recovery scanner dependencies and options are required")
|
||||
}
|
||||
return &RecoveryScanner{pool: pool, outbox: outboxStore, logger: logger, options: options, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (s *RecoveryScanner) Run(ctx context.Context) {
|
||||
s.logger.Info("conversation recovery scanner started")
|
||||
ticker := time.NewTicker(s.options.ScanInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if count, err := s.ScanOnce(ctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||
s.logger.Error("conversation recovery scan failed", "error", err)
|
||||
} else if count > 0 {
|
||||
s.logger.Info("conversation recovery actions scheduled", "count", count)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
s.logger.Info("conversation recovery scanner stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *RecoveryScanner) ScanOnce(ctx context.Context) (int, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
now := s.now().UTC()
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT c.id::text, c.public_id::text, c.tenant_id::text, c.status,
|
||||
c.provider_task_ref, COALESCE(c.stop_reason,''),
|
||||
CASE WHEN c.status='ACTIVE' AND cub.in_progress_action='STOP_IMMEDIATELY'
|
||||
AND c.started_at IS NOT NULL
|
||||
AND c.started_at+make_interval(secs=>cub.max_call_duration_sec)<=$1
|
||||
THEN 'quota_duration_limit' ELSE '' END
|
||||
FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id
|
||||
LEFT JOIN conversation_usage_budget cub ON cub.conversation_id=c.id AND cub.tenant_id=c.tenant_id
|
||||
WHERE (c.status='ROOM_ISSUED' AND c.allocation_expires_at <= $1)
|
||||
OR (c.status='ACTIVE' AND c.lease_expires_at IS NOT NULL AND c.lease_expires_at <= $1)
|
||||
OR (c.status='ACTIVE' AND cub.in_progress_action='STOP_IMMEDIATELY' AND c.started_at IS NOT NULL AND c.started_at+make_interval(secs=>cub.max_call_duration_sec)<=$1)
|
||||
OR (c.status='STARTING_AGENT' AND c.updated_at <= $2)
|
||||
OR (c.status='STOPPING' AND c.updated_at <= $3)
|
||||
ORDER BY c.updated_at, c.id
|
||||
LIMIT $4 FOR UPDATE OF c, r SKIP LOCKED`,
|
||||
now, now.Add(-s.options.StartingAfter), now.Add(-s.options.StoppingAfter), s.options.BatchSize)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("select conversations for recovery: %w", err)
|
||||
}
|
||||
candidates := make([]recoveryCandidate, 0, s.options.BatchSize)
|
||||
for rows.Next() {
|
||||
var candidate recoveryCandidate
|
||||
if err := rows.Scan(&candidate.InternalID, &candidate.PublicID, &candidate.TenantID, &candidate.Status, &candidate.ProviderTaskRef, &candidate.StopReason, &candidate.Trigger); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
candidates = append(candidates, candidate)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
rows.Close()
|
||||
for _, candidate := range candidates {
|
||||
if err := s.recoverCandidate(ctx, tx, candidate, now); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(candidates), nil
|
||||
}
|
||||
|
||||
func (s *RecoveryScanner) recoverCandidate(ctx context.Context, tx pgx.Tx, candidate recoveryCandidate, now time.Time) error {
|
||||
action := ""
|
||||
switch candidate.Status {
|
||||
case conversation.StatusRoomIssued:
|
||||
action = "expire_unactivated_room"
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET status='EXPIRED', ended_at=$2,
|
||||
failure_code='activation_timeout', lease_expires_at=NULL, version=version+1, updated_at=$2
|
||||
WHERE id=$1::uuid AND status='ROOM_ISSUED'`, candidate.InternalID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED', lease_expires_at=NULL,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, candidate.InternalID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source
|
||||
) VALUES ($1::uuid, 'ROOM_ISSUED', 'EXPIRED', 'activation deadline expired', 'worker')`, candidate.InternalID); err != nil {
|
||||
return err
|
||||
}
|
||||
case conversation.StatusActive:
|
||||
action = "stop_expired_lease"
|
||||
reason := "lease_timeout"
|
||||
historyReason := "conversation lease expired"
|
||||
if candidate.Trigger == "quota_duration_limit" {
|
||||
action = "stop_quota_duration_limit"
|
||||
reason = "quota_duration_limit"
|
||||
historyReason = "maximum call duration reached"
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET status='STOPPING', stop_reason=$3,
|
||||
lease_expires_at=NULL, version=version+1, updated_at=$2
|
||||
WHERE id=$1::uuid AND status='ACTIVE'`, candidate.InternalID, now, reason); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='STOPPING', lease_expires_at=NULL,
|
||||
updated_at=$2 WHERE conversation_id=$1::uuid`, candidate.InternalID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source
|
||||
) VALUES ($1::uuid, 'ACTIVE', 'STOPPING', $2, 'worker')`, candidate.InternalID, historyReason); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.enqueueRecovery(ctx, tx, candidate, "conversation.agent_stop_requested", candidate.ProviderTaskRef, reason); err != nil {
|
||||
return err
|
||||
}
|
||||
case conversation.StatusStartingAgent:
|
||||
action = "reconcile_starting_task"
|
||||
deduplicationKey := candidate.ProviderTaskRef + ":" + strconv.FormatInt(recoveryBucket(now, s.options.StartingAfter), 10)
|
||||
if err := s.enqueueRecovery(ctx, tx, candidate, "conversation.agent_start_reconcile_requested", deduplicationKey, "starting_timeout"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET updated_at=$2 WHERE id=$1::uuid`, candidate.InternalID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
case conversation.StatusStopping:
|
||||
action = "reconcile_stopping_task"
|
||||
deduplicationKey := candidate.ProviderTaskRef + ":" + strconv.FormatInt(recoveryBucket(now, s.options.StoppingAfter), 10)
|
||||
reason := candidate.StopReason
|
||||
if reason == "" {
|
||||
reason = "recovery_stop"
|
||||
}
|
||||
if err := s.enqueueRecovery(ctx, tx, candidate, "conversation.agent_stop_reconcile_requested", deduplicationKey, reason); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET updated_at=$2 WHERE id=$1::uuid`, candidate.InternalID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
detail, _ := json.Marshal(map[string]any{"previous_status": candidate.Status, "action": action})
|
||||
_, err := tx.Exec(ctx, `INSERT INTO audit_event (
|
||||
tenant_id, actor_type, action, resource_type, resource_id, purpose, result, detail
|
||||
) VALUES ($1::uuid, 'system', 'conversation.recovery', 'conversation', $2::uuid, $3, 'success', $4::jsonb)`,
|
||||
candidate.TenantID, candidate.InternalID, action, detail)
|
||||
return err
|
||||
}
|
||||
|
||||
func recoveryBucket(now time.Time, interval time.Duration) int64 {
|
||||
bucketWidth := int64(interval / time.Second)
|
||||
if bucketWidth < 1 {
|
||||
bucketWidth = 1
|
||||
}
|
||||
return now.Unix() / bucketWidth
|
||||
}
|
||||
|
||||
func (s *RecoveryScanner) enqueueRecovery(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
candidate recoveryCandidate,
|
||||
eventType, deduplicationKey, reason string,
|
||||
) error {
|
||||
payload, _ := json.Marshal(map[string]string{
|
||||
"conversation_id": candidate.PublicID, "provider_task_ref": candidate.ProviderTaskRef, "reason": reason,
|
||||
})
|
||||
_, err := s.outbox.Enqueue(ctx, tx, outbox.NewEvent{
|
||||
TenantID: &candidate.TenantID, AggregateType: "conversation", AggregateID: candidate.InternalID,
|
||||
EventType: eventType, DeduplicationKey: &deduplicationKey, Payload: payload,
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/deviceauth"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
"data-terminal-cloud/internal/voiceprovider"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestRecoveryScannerExpiresAndSchedulesIdempotentReconciliation(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "orchestration-recovery-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
identity := deviceauth.Identity{
|
||||
ID: "10000000-0000-0000-0000-000000000501", PublicID: "10000000-0000-0000-0000-000000000511",
|
||||
TenantID: "10000000-0000-0000-0000-000000000001", Status: "registered",
|
||||
}
|
||||
now := time.Date(2026, 8, 23, 13, 0, 0, 0, time.UTC)
|
||||
publicIDs := make([]string, 0, 3)
|
||||
defer func() { cleanupRecoveryConversations(t, context.Background(), pool, publicIDs) }()
|
||||
|
||||
roomIssued, roomInternalID := createRecoveryConversation(t, ctx, pool, identity, now.Add(-time.Minute), now.Add(-30*time.Second))
|
||||
publicIDs = append(publicIDs, roomIssued)
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
scanner, err := NewRecoveryScanner(pool, outbox.NewStore(pool), logger, RecoveryOptions{
|
||||
ScanInterval: time.Second, StartingAfter: 10 * time.Second, StoppingAfter: 10 * time.Second, BatchSize: 10,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanner.now = func() time.Time { return now }
|
||||
if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 {
|
||||
t.Fatalf("expired room scan count=%d err=%v", count, err)
|
||||
}
|
||||
var status, taskStatus string
|
||||
if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.id=$1::uuid`, roomInternalID).Scan(&status, &taskStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "EXPIRED" || taskStatus != "COMPLETED" {
|
||||
t.Fatalf("expired room conversation=%s task=%s", status, taskStatus)
|
||||
}
|
||||
|
||||
activePublicID, activeInternalID := createRecoveryConversation(t, ctx, pool, identity, now, now.Add(time.Minute))
|
||||
publicIDs = append(publicIDs, activePublicID)
|
||||
if _, err := pool.Exec(ctx, `UPDATE conversation SET status='ACTIVE', lease_expires_at=$2, updated_at=$3 WHERE id=$1::uuid`, activeInternalID, now.Add(-time.Second), now.Add(-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='ACTIVE', lease_expires_at=$2 WHERE conversation_id=$1::uuid`, activeInternalID, now.Add(-time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 {
|
||||
t.Fatalf("expired lease scan count=%d err=%v", count, err)
|
||||
}
|
||||
var stopEvents int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=$1::uuid AND event_type='conversation.agent_stop_requested'`, activeInternalID).Scan(&stopEvents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.id=$1::uuid`, activeInternalID).Scan(&status, &taskStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "STOPPING" || taskStatus != "STOPPING" || stopEvents != 1 {
|
||||
t.Fatalf("lease recovery conversation=%s task=%s stopEvents=%d", status, taskStatus, stopEvents)
|
||||
}
|
||||
scanner.now = func() time.Time { return now.Add(11 * time.Second) }
|
||||
if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 {
|
||||
t.Fatalf("stopping reconciliation count=%d err=%v", count, err)
|
||||
}
|
||||
var stopReconcileEvents int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=$1::uuid AND event_type='conversation.agent_stop_reconcile_requested'`, activeInternalID).Scan(&stopReconcileEvents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stopReconcileEvents != 1 {
|
||||
t.Fatalf("stop reconcile events=%d", stopReconcileEvents)
|
||||
}
|
||||
|
||||
if _, err := pool.Exec(ctx, `UPDATE conversation SET status='COMPLETED', ended_at=$2 WHERE id=$1::uuid`, activeInternalID, now.Add(12*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED' WHERE conversation_id=$1::uuid`, activeInternalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
startingPublicID, startingInternalID := createRecoveryConversation(t, ctx, pool, identity, now, now.Add(time.Minute))
|
||||
publicIDs = append(publicIDs, startingPublicID)
|
||||
if _, err := pool.Exec(ctx, `UPDATE conversation SET status='STARTING_AGENT', updated_at=$2 WHERE id=$1::uuid`, startingInternalID, now.Add(-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='STARTING' WHERE conversation_id=$1::uuid`, startingInternalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 {
|
||||
t.Fatalf("starting reconciliation count=%d err=%v", count, err)
|
||||
}
|
||||
var startReconcileEvents, recoveryAudits int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=$1::uuid AND event_type='conversation.agent_start_reconcile_requested'`, startingInternalID).Scan(&startReconcileEvents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM audit_event WHERE action='conversation.recovery' AND resource_id IN ($1::uuid,$2::uuid,$3::uuid)`, roomInternalID, activeInternalID, startingInternalID).Scan(&recoveryAudits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if startReconcileEvents != 1 || recoveryAudits != 4 {
|
||||
t.Fatalf("start reconcile=%d recovery audits=%d", startReconcileEvents, recoveryAudits)
|
||||
}
|
||||
var providerTaskRef string
|
||||
if err := pool.QueryRow(ctx, `SELECT provider_task_ref FROM conversation WHERE id=$1::uuid`, startingInternalID).Scan(&providerTaskRef); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
processor, err := NewProviderEventProcessor(pool, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
processor.now = func() time.Time { return now.Add(12 * time.Second) }
|
||||
failedEvent := voiceprovider.CallbackEvent{
|
||||
Provider: "mock", EventID: "recovery-test-provider-failed", TaskRef: providerTaskRef,
|
||||
ConversationID: startingPublicID, EventType: "voice_chat.failed", Status: "FAILED",
|
||||
OccurredAt: now.Add(12 * time.Second),
|
||||
}
|
||||
failedBody, _ := json.Marshal(failedEvent)
|
||||
failedResult, err := processor.Process(ctx, failedEvent, failedBody)
|
||||
if err != nil || failedResult.Status != "FAILED" || failedResult.Ignored {
|
||||
t.Fatalf("failed callback result=%#v err=%v", failedResult, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.id=$1::uuid`, startingInternalID).Scan(&status, &taskStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "FAILED" || taskStatus != "FAILED" {
|
||||
t.Fatalf("failed callback conversation=%s task=%s", status, taskStatus)
|
||||
}
|
||||
canceledContext, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
scanner.Run(canceledContext)
|
||||
}
|
||||
|
||||
func createRecoveryConversation(t *testing.T, ctx context.Context, pool *pgxpool.Pool, identity deviceauth.Identity, now, allocationExpires time.Time) (string, string) {
|
||||
t.Helper()
|
||||
tx, err := pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
resume := true
|
||||
response, internalID, err := NewRepository().Create(ctx, tx, identity, CreateInput{
|
||||
DeviceTime: now, Capabilities: Capabilities{RTC: true, AudioTracks: []string{"near"}, UploadResume: &resume},
|
||||
Network: Network{Type: "test"}, RequestedGoal: "open_conversation",
|
||||
}, AllocationOptions{
|
||||
Now: now, AllocationExpires: allocationExpires, TokenExpires: now.Add(time.Minute),
|
||||
RTCAppID: "test-app", EngineMode: "s2s_hybrid", HeartbeatInterval: 15,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return response.ConversationID, internalID
|
||||
}
|
||||
|
||||
func cleanupRecoveryConversations(t *testing.T, ctx context.Context, pool *pgxpool.Pool, publicIDs []string) {
|
||||
t.Helper()
|
||||
for _, publicID := range publicIDs {
|
||||
var internalID, contextSnapshotID, consentSnapshotID string
|
||||
err := pool.QueryRow(ctx, `SELECT id::text, context_snapshot_id::text, consent_snapshot_id::text FROM conversation WHERE public_id=$1::uuid`, publicID).Scan(&internalID, &contextSnapshotID, &consentSnapshotID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
statements := []string{
|
||||
`DELETE FROM provider_event WHERE conversation_id=$1::uuid`,
|
||||
`DELETE FROM conversation_state_history WHERE conversation_id=$1::uuid`,
|
||||
`DELETE FROM rtc_task WHERE conversation_id=$1::uuid`,
|
||||
`DELETE FROM outbox_event WHERE aggregate_id=$1::uuid`,
|
||||
`DELETE FROM audit_event WHERE resource_id=$1::uuid`,
|
||||
`DELETE FROM idempotency_record WHERE resource_id=$1::uuid`,
|
||||
`DELETE FROM conversation WHERE id=$1::uuid`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := pool.Exec(ctx, statement, internalID); err != nil {
|
||||
t.Errorf("cleanup recovery conversation: %v", err)
|
||||
}
|
||||
}
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM context_snapshot WHERE id=$1::uuid`, contextSnapshotID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM consent_snapshot WHERE id=$1::uuid`, consentSnapshotID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,786 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/billing"
|
||||
"data-terminal-cloud/internal/conversation"
|
||||
"data-terminal-cloud/internal/deviceauth"
|
||||
"data-terminal-cloud/internal/firmwareversion"
|
||||
"data-terminal-cloud/internal/retrieval"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrDeviceNotBound = errors.New("device is not bound to an active subject")
|
||||
ErrConsentRequired = errors.New("voice chat consent is required")
|
||||
ErrFirmwareUpdateRequired = errors.New("device firmware does not satisfy the minimum version")
|
||||
ErrConversationAlreadyActive = errors.New("device already has an unfinished conversation")
|
||||
ErrConversationNotFound = errors.New("conversation was not found for device")
|
||||
ErrConversationStateConflict = errors.New("conversation state does not allow operation")
|
||||
ErrAllocationExpired = errors.New("room allocation expired")
|
||||
ErrRTCIdentityMismatch = errors.New("RTC room or user does not match allocation")
|
||||
ErrConfigVersionConflict = errors.New("device configuration revision is stale")
|
||||
ErrRuntimeConfigUnavailable = errors.New("published runtime routing configuration is unavailable")
|
||||
ErrContextPreparationFailed = errors.New("authorized subject context preparation failed")
|
||||
ErrUsageQuotaExceeded = billing.ErrQuotaExceeded
|
||||
)
|
||||
|
||||
const contextPreparationFailureCode = "context_preparation_failed"
|
||||
|
||||
type AllocationOptions struct {
|
||||
Now time.Time
|
||||
AllocationExpires time.Time
|
||||
TokenExpires time.Time
|
||||
RTCAppID string
|
||||
VoiceProvider string
|
||||
EngineMode string
|
||||
HeartbeatInterval int
|
||||
MemoryQueryVector string
|
||||
MemoryPolicy string
|
||||
RequirePublishedRouting bool
|
||||
}
|
||||
|
||||
type Repository struct{}
|
||||
|
||||
type ActivationResult struct {
|
||||
Response ActivateResponse
|
||||
InternalID string
|
||||
ProviderTaskRef string
|
||||
StartRequested bool
|
||||
}
|
||||
|
||||
type StopResult struct {
|
||||
Response StopResponse
|
||||
InternalID string
|
||||
ProviderTaskRef string
|
||||
StopRequested bool
|
||||
}
|
||||
|
||||
func NewRepository() *Repository { return &Repository{} }
|
||||
|
||||
func (r *Repository) Stop(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
identity deviceauth.Identity,
|
||||
conversationPublicID, requestID string,
|
||||
input StopInput,
|
||||
now time.Time,
|
||||
) (StopResult, error) {
|
||||
var internalID, providerTaskRef string
|
||||
var status conversation.Status
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT c.id::text, c.status, c.provider_task_ref
|
||||
FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id
|
||||
WHERE c.public_id=$1::uuid AND c.device_id=$2::uuid AND c.tenant_id=$3::uuid
|
||||
FOR UPDATE OF c, r`, conversationPublicID, identity.ID, identity.TenantID,
|
||||
).Scan(&internalID, &status, &providerTaskRef)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return StopResult{}, ErrConversationNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return StopResult{}, fmt.Errorf("read conversation stop state: %w", err)
|
||||
}
|
||||
result := StopResult{
|
||||
Response: StopResponse{ConversationID: conversationPublicID, Status: status, AcceptedAt: now},
|
||||
InternalID: internalID, ProviderTaskRef: providerTaskRef,
|
||||
}
|
||||
if conversation.IsTerminal(status) {
|
||||
return result, nil
|
||||
}
|
||||
if status == conversation.StatusStopping {
|
||||
_, err := tx.Exec(ctx, `
|
||||
UPDATE conversation SET stop_reason=COALESCE(stop_reason,$2),
|
||||
last_heartbeat_seq=GREATEST(last_heartbeat_seq,$3), updated_at=$4
|
||||
WHERE id=$1::uuid`, internalID, input.Reason, input.LastHeartbeatSequence, now)
|
||||
if err != nil {
|
||||
return StopResult{}, fmt.Errorf("refresh repeated stop: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
if status == conversation.StatusRoomIssued {
|
||||
if err := conversation.ValidateTransition(status, conversation.StatusExpired); err != nil {
|
||||
return StopResult{}, err
|
||||
}
|
||||
_, err := tx.Exec(ctx, `
|
||||
UPDATE conversation SET status='EXPIRED', stop_reason=$2, ended_at=$3,
|
||||
lease_expires_at=NULL, last_heartbeat_seq=GREATEST(last_heartbeat_seq,$4),
|
||||
version=version+1, updated_at=$3 WHERE id=$1::uuid AND status='ROOM_ISSUED'`,
|
||||
internalID, input.Reason, now, input.LastHeartbeatSequence)
|
||||
if err == nil {
|
||||
_, err = tx.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED', lease_expires_at=NULL,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, internalID, now)
|
||||
}
|
||||
if err == nil {
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, request_id
|
||||
) VALUES ($1::uuid, 'ROOM_ISSUED', 'EXPIRED', 'device stopped before activation', 'device', NULLIF($2,''))`,
|
||||
internalID, requestID)
|
||||
}
|
||||
if err != nil {
|
||||
return StopResult{}, fmt.Errorf("expire unactivated conversation: %w", err)
|
||||
}
|
||||
result.Response.Status = conversation.StatusExpired
|
||||
return result, nil
|
||||
}
|
||||
if status != conversation.StatusStartingAgent && status != conversation.StatusActive {
|
||||
return StopResult{}, ErrConversationStateConflict
|
||||
}
|
||||
if err := conversation.ValidateTransition(status, conversation.StatusStopping); err != nil {
|
||||
return StopResult{}, err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE conversation SET status='STOPPING', stop_reason=$2,
|
||||
lease_expires_at=NULL, last_heartbeat_seq=GREATEST(last_heartbeat_seq,$3),
|
||||
version=version+1, updated_at=$4 WHERE id=$1::uuid AND status=$5`,
|
||||
internalID, input.Reason, input.LastHeartbeatSequence, now, status)
|
||||
if err == nil {
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE rtc_task SET status='STOPPING', lease_expires_at=NULL, updated_at=$2
|
||||
WHERE conversation_id=$1::uuid`, internalID, now)
|
||||
}
|
||||
if err == nil {
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, request_id
|
||||
) VALUES ($1::uuid, $2, 'STOPPING', 'device requested conversation stop', 'device', NULLIF($3,''))`,
|
||||
internalID, status, requestID)
|
||||
}
|
||||
if err != nil {
|
||||
return StopResult{}, fmt.Errorf("stop active conversation: %w", err)
|
||||
}
|
||||
result.Response.Status = conversation.StatusStopping
|
||||
result.StopRequested = true
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r *Repository) Activate(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
identity deviceauth.Identity,
|
||||
conversationPublicID, requestID string,
|
||||
input ActivateInput,
|
||||
now time.Time,
|
||||
) (ActivationResult, error) {
|
||||
var internalID, providerTaskRef, roomID, rtcUserID string
|
||||
var status conversation.Status
|
||||
var allocationExpires time.Time
|
||||
var desiredRevision int64
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT c.id::text, c.status, c.allocation_expires_at, c.provider_task_ref,
|
||||
r.room_id, r.device_rtc_user_id, dt.desired_revision
|
||||
FROM conversation c
|
||||
JOIN rtc_task r ON r.conversation_id = c.id
|
||||
JOIN device_twin dt ON dt.device_id = c.device_id AND dt.tenant_id = c.tenant_id
|
||||
JOIN device_binding b ON b.device_id = c.device_id AND b.tenant_id = c.tenant_id
|
||||
AND b.subject_id = c.subject_id AND b.unbound_at IS NULL
|
||||
WHERE c.public_id = $1::uuid AND c.device_id = $2::uuid AND c.tenant_id = $3::uuid
|
||||
FOR UPDATE OF c, r`, conversationPublicID, identity.ID, identity.TenantID,
|
||||
).Scan(&internalID, &status, &allocationExpires, &providerTaskRef, &roomID, &rtcUserID, &desiredRevision)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ActivationResult{}, ErrConversationNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return ActivationResult{}, fmt.Errorf("read conversation activation state: %w", err)
|
||||
}
|
||||
if input.RoomID != roomID || input.RTCUserID != rtcUserID {
|
||||
return ActivationResult{}, ErrRTCIdentityMismatch
|
||||
}
|
||||
if input.RTCJoinResult != "success" || input.JoinedAt.IsZero() || input.JoinedAt.After(now.Add(5*time.Minute)) {
|
||||
return ActivationResult{}, ErrRTCIdentityMismatch
|
||||
}
|
||||
if input.ConfigRevision != desiredRevision {
|
||||
return ActivationResult{}, ErrConfigVersionConflict
|
||||
}
|
||||
result := ActivationResult{
|
||||
Response: ActivateResponse{ConversationID: conversationPublicID, Status: status, AcceptedAt: now},
|
||||
InternalID: internalID, ProviderTaskRef: providerTaskRef,
|
||||
}
|
||||
if status == conversation.StatusStartingAgent || status == conversation.StatusActive {
|
||||
return result, nil
|
||||
}
|
||||
if status != conversation.StatusRoomIssued {
|
||||
return ActivationResult{}, ErrConversationStateConflict
|
||||
}
|
||||
if !allocationExpires.After(now) {
|
||||
return ActivationResult{}, ErrAllocationExpired
|
||||
}
|
||||
if err := conversation.ValidateTransition(status, conversation.StatusStartingAgent); err != nil {
|
||||
return ActivationResult{}, err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE conversation
|
||||
SET status='STARTING_AGENT', version=version+1, updated_at=$2
|
||||
WHERE id=$1::uuid AND status='ROOM_ISSUED'`, internalID, now)
|
||||
if err != nil {
|
||||
return ActivationResult{}, fmt.Errorf("start conversation activation: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return ActivationResult{}, ErrConversationStateConflict
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE rtc_task SET status='STARTING', updated_at=$2
|
||||
WHERE conversation_id=$1::uuid AND status='ALLOCATED'`, internalID, now)
|
||||
if err != nil {
|
||||
return ActivationResult{}, fmt.Errorf("mark RTC task starting: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, request_id
|
||||
) VALUES ($1::uuid, 'ROOM_ISSUED', 'STARTING_AGENT', 'device confirmed RTC join', 'device', NULLIF($2,''))`,
|
||||
internalID, requestID)
|
||||
if err != nil {
|
||||
return ActivationResult{}, fmt.Errorf("record activation state: %w", err)
|
||||
}
|
||||
result.Response.Status = conversation.StatusStartingAgent
|
||||
result.StartRequested = true
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r *Repository) Create(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
identity deviceauth.Identity,
|
||||
input CreateInput,
|
||||
options AllocationOptions,
|
||||
) (CreateResponse, string, error) {
|
||||
if options.VoiceProvider == "" {
|
||||
options.VoiceProvider = "mock"
|
||||
}
|
||||
var householdID, householdPublicID, subjectID, subjectPublicID, subjectName string
|
||||
var profileDataJSON, conversationPreferencesJSON json.RawMessage
|
||||
var minimumFirmwareVersion, currentFirmwareVersion *string
|
||||
var profileVersion int64
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT b.household_id::text, h.public_id::text, b.subject_id::text,
|
||||
s.public_id::text, s.preferred_name, s.profile_data, s.conversation_preferences, s.profile_version,
|
||||
d.minimum_firmware_version, dt.firmware_version
|
||||
FROM device d
|
||||
JOIN device_twin dt ON dt.device_id=d.id AND dt.tenant_id=d.tenant_id
|
||||
JOIN device_binding b ON b.device_id = d.id AND b.tenant_id = d.tenant_id AND b.unbound_at IS NULL
|
||||
JOIN household h ON h.id = b.household_id AND h.tenant_id = b.tenant_id AND h.status = 'active'
|
||||
JOIN subject_profile s ON s.id = b.subject_id AND s.tenant_id = b.tenant_id AND s.status = 'active'
|
||||
WHERE d.id = $1::uuid AND d.tenant_id = $2::uuid AND d.status = 'registered'
|
||||
FOR UPDATE OF d, b, s, dt`, identity.ID, identity.TenantID,
|
||||
).Scan(&householdID, &householdPublicID, &subjectID, &subjectPublicID, &subjectName,
|
||||
&profileDataJSON, &conversationPreferencesJSON, &profileVersion, &minimumFirmwareVersion, ¤tFirmwareVersion)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return CreateResponse{}, "", ErrDeviceNotBound
|
||||
}
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("read active device binding: %w", err)
|
||||
}
|
||||
if minimumFirmwareVersion != nil && (currentFirmwareVersion == nil || !firmwareversion.AtLeast(*currentFirmwareVersion, *minimumFirmwareVersion)) {
|
||||
return CreateResponse{}, "", ErrFirmwareUpdateRequired
|
||||
}
|
||||
var activeConversationExists bool
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM conversation
|
||||
WHERE device_id=$1::uuid AND status NOT IN ('COMPLETED','FAILED','EXPIRED')
|
||||
)`, identity.ID).Scan(&activeConversationExists); err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("check active device conversation: %w", err)
|
||||
}
|
||||
if activeConversationExists {
|
||||
return CreateResponse{}, "", ErrConversationAlreadyActive
|
||||
}
|
||||
budgetSnapshot, err := billing.CheckQuotaTx(ctx, tx, identity, options.Now)
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("check usage quota: %w", err)
|
||||
}
|
||||
engineMode := options.EngineMode
|
||||
promptBundleVersion := "minimal-v2"
|
||||
publishedConfigID, configuredEngineMode, err := selectRuntimeConfigurationTx(ctx, tx, identity.TenantID, subjectID, identity.ID, identity.PublicID)
|
||||
if err == nil {
|
||||
if configuredEngineMode != "s2s_pure" && configuredEngineMode != "s2s_hybrid" && configuredEngineMode != "cascade" {
|
||||
return CreateResponse{}, "", fmt.Errorf("published routing configuration contains an invalid engine mode")
|
||||
}
|
||||
engineMode = configuredEngineMode
|
||||
promptBundleVersion = "cfg:" + publishedConfigID
|
||||
} else if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return CreateResponse{}, "", fmt.Errorf("read published runtime configuration: %w", err)
|
||||
} else if options.RequirePublishedRouting {
|
||||
return CreateResponse{}, "", ErrRuntimeConfigUnavailable
|
||||
}
|
||||
|
||||
var sourceConsentPublicID string
|
||||
var snapshotVersion int64
|
||||
var effectiveConsents, sourceConsentIDs json.RawMessage
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT public_id::text, snapshot_version, effective_consents, source_consent_ids
|
||||
FROM consent_snapshot
|
||||
WHERE tenant_id = $1::uuid AND subject_id = $2::uuid
|
||||
AND effective_consents @> '{"voice_chat":true}'::jsonb
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT 1`, identity.TenantID, subjectID,
|
||||
).Scan(&sourceConsentPublicID, &snapshotVersion, &effectiveConsents, &sourceConsentIDs)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return CreateResponse{}, "", ErrConsentRequired
|
||||
}
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("read effective consent: %w", err)
|
||||
}
|
||||
|
||||
consentSnapshotID := uuid.NewString()
|
||||
consentSnapshotPublicID := uuid.NewString()
|
||||
traceSources, _ := json.Marshal([]string{sourceConsentPublicID})
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO consent_snapshot (
|
||||
id, public_id, tenant_id, subject_id, snapshot_version,
|
||||
effective_consents, source_consent_ids
|
||||
) VALUES ($1::uuid, $2::uuid, $3::uuid, $4::uuid, $5, $6::jsonb, $7::jsonb)`,
|
||||
consentSnapshotID, consentSnapshotPublicID, identity.TenantID, subjectID,
|
||||
snapshotVersion, effectiveConsents, traceSources,
|
||||
)
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("freeze consent snapshot: %w", err)
|
||||
}
|
||||
|
||||
var memories []retrieval.Memory
|
||||
if options.MemoryPolicy == "" {
|
||||
memories, err = retrieval.SelectMemoryForContext(ctx, tx, identity.TenantID, subjectID, subjectName+" "+input.RequestedGoal, 5)
|
||||
} else if options.MemoryQueryVector != "" {
|
||||
memories, err = retrieval.SelectMemoryForContextVector(ctx, tx, identity.TenantID, subjectID, options.MemoryQueryVector, 5)
|
||||
}
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("retrieve conversation memory: %w", err)
|
||||
}
|
||||
memoryVersionsJSON, _ := json.Marshal(memories)
|
||||
memoryContents := make([]string, 0, len(memories))
|
||||
for _, memory := range memories {
|
||||
memoryContents = append(memoryContents, truncateContextText(memory.Content, 300))
|
||||
}
|
||||
profileContext, err := authorizedProfileContext(profileDataJSON, conversationPreferencesJSON)
|
||||
if err != nil {
|
||||
return r.createContextPreparationFailure(ctx, tx, identity, householdID, subjectID, profileVersion,
|
||||
consentSnapshotID, memoryVersionsJSON, promptBundleVersion, engineMode,
|
||||
privacyFromConsent(effectiveConsents), options)
|
||||
}
|
||||
contextSummary := buildContextSummary(subjectName, profileContext)
|
||||
if len(memoryContents) > 0 {
|
||||
contextSummary += "；已确认记忆：" + strings.Join(memoryContents, "；")
|
||||
}
|
||||
contextValue := struct {
|
||||
SubjectID string `json:"subject_id"`
|
||||
PreferredName string `json:"preferred_name"`
|
||||
ProfileVersion int64 `json:"profile_version"`
|
||||
RequestedGoal string `json:"requested_goal"`
|
||||
ProfileContext authorizedSubjectContext `json:"profile_context"`
|
||||
MemoryContents []string `json:"memory_contents"`
|
||||
RetrievalPolicy string `json:"retrieval_policy"`
|
||||
}{subjectPublicID, subjectName, profileVersion, input.RequestedGoal, profileContext, memoryContents, retrievalPolicy(options.MemoryPolicy)}
|
||||
contextJSON, _ := json.Marshal(contextValue)
|
||||
contextDigest := sha256.Sum256(contextJSON)
|
||||
contextSnapshotID := uuid.NewString()
|
||||
contextSnapshotPublicID := uuid.NewString()
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO context_snapshot (
|
||||
id, public_id, tenant_id, subject_id, profile_version, consent_snapshot_id,
|
||||
memory_versions, prompt_bundle_version, engine_mode, content_digest, sanitized_summary
|
||||
) VALUES ($1::uuid, $2::uuid, $3::uuid, $4::uuid, $5, $6::uuid,
|
||||
$7::jsonb, $8, $9, $10, $11)`,
|
||||
contextSnapshotID, contextSnapshotPublicID, identity.TenantID, subjectID, profileVersion,
|
||||
consentSnapshotID, memoryVersionsJSON, promptBundleVersion, engineMode, hex.EncodeToString(contextDigest[:]), contextSummary,
|
||||
)
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("create context snapshot: %w", err)
|
||||
}
|
||||
|
||||
conversationID := uuid.NewString()
|
||||
conversationPublicID := uuid.NewString()
|
||||
providerTaskRef := "voice-task-" + uuid.NewString()
|
||||
roomID := "r_" + strings.ReplaceAll(uuid.NewString(), "-", "")
|
||||
deviceRTCUserID := "d_" + strings.ReplaceAll(identity.PublicID, "-", "")
|
||||
agentRTCUserID := "a_" + strings.ReplaceAll(conversationPublicID, "-", "")
|
||||
privacy := privacyFromConsent(effectiveConsents)
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO conversation (
|
||||
id, public_id, tenant_id, household_id, subject_id, device_id,
|
||||
consent_snapshot_id, context_snapshot_id, provider_task_ref,
|
||||
engine_mode, retention_mode, rtc_credential_renewal_supported,
|
||||
status, allocation_expires_at, created_at, updated_at
|
||||
) VALUES (
|
||||
$1::uuid, $2::uuid, $3::uuid, $4::uuid, $5::uuid, $6::uuid,
|
||||
$7::uuid, $8::uuid, $9, $10, $11, COALESCE($12, false), 'PREPARING', $13, $14, $14
|
||||
)`, conversationID, conversationPublicID, identity.TenantID, householdID, subjectID, identity.ID,
|
||||
consentSnapshotID, contextSnapshotID, providerTaskRef, engineMode,
|
||||
privacy.RetentionMode, input.Capabilities.RTCCredentialRenewal, options.AllocationExpires, options.Now,
|
||||
)
|
||||
if err != nil {
|
||||
var postgresError *pgconn.PgError
|
||||
if errors.As(err, &postgresError) && postgresError.ConstraintName == "conversation_one_active_per_device" {
|
||||
return CreateResponse{}, "", ErrConversationAlreadyActive
|
||||
}
|
||||
return CreateResponse{}, "", fmt.Errorf("create conversation: %w", err)
|
||||
}
|
||||
if err = billing.FreezeBudgetTx(ctx, tx, conversationID, identity.TenantID, budgetSnapshot); err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("freeze conversation usage budget: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (conversation_id, from_status, to_status, reason, source)
|
||||
VALUES ($1::uuid, NULL, 'PREPARING', 'create request accepted', 'device')`, conversationID)
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("record preparing state: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO rtc_task (
|
||||
conversation_id, provider, provider_task_ref, room_id,
|
||||
device_rtc_user_id, agent_rtc_user_id, status, lease_expires_at,
|
||||
credential_issued_at, credential_expires_at
|
||||
) VALUES ($1::uuid, $2, $3, $4, $5, $6, 'ALLOCATED', $7, $8, $9)`,
|
||||
conversationID, options.VoiceProvider, providerTaskRef, roomID, deviceRTCUserID, agentRTCUserID,
|
||||
options.AllocationExpires, options.Now, options.TokenExpires,
|
||||
)
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("allocate RTC task: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE conversation SET status = 'ROOM_ISSUED', version = version + 1, updated_at = $2
|
||||
WHERE id = $1::uuid AND status = 'PREPARING'`, conversationID, options.Now)
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("issue conversation room: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (conversation_id, from_status, to_status, reason, source)
|
||||
VALUES ($1::uuid, 'PREPARING', 'ROOM_ISSUED', 'room credentials allocated', 'platform')`, conversationID)
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("record room issued state: %w", err)
|
||||
}
|
||||
|
||||
_ = householdPublicID
|
||||
return CreateResponse{
|
||||
ConversationID: conversationPublicID,
|
||||
Status: conversation.StatusRoomIssued,
|
||||
RTC: RTCGrant{
|
||||
AppID: options.RTCAppID, RoomID: roomID, UserID: deviceRTCUserID,
|
||||
AgentUserID: agentRTCUserID, ExpiresAt: options.TokenExpires,
|
||||
},
|
||||
Privacy: privacy,
|
||||
Session: Session{
|
||||
EngineMode: engineMode, HeartbeatIntervalSecond: options.HeartbeatInterval,
|
||||
AllocationExpiresAt: options.AllocationExpires,
|
||||
},
|
||||
}, conversationID, nil
|
||||
}
|
||||
|
||||
func selectRuntimeConfigurationTx(ctx context.Context, tx pgx.Tx, tenantID, subjectID, deviceID, devicePublicID string) (string, string, error) {
|
||||
// Rollouts already bind an immutable configuration version and its business
|
||||
// type. ConfigKey is an operator-defined version family, not a runtime
|
||||
// discriminator: filtering on a reserved key would make a successfully
|
||||
// published custom Agent bundle silently ineffective. Prefer the newest
|
||||
// matching Agent bundle at the most specific scope, then retain any direct
|
||||
// ROUTING rollout as the compatibility path for installations that have not
|
||||
// adopted bundles yet.
|
||||
var configID, engineMode string
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT bundle.public_id::text,routing.payload->>'default_engine_mode'
|
||||
FROM runtime_config_rollout rr
|
||||
JOIN runtime_config_version bundle
|
||||
ON bundle.id=rr.config_version_id AND bundle.tenant_id=rr.tenant_id
|
||||
JOIN runtime_config_version routing
|
||||
ON routing.tenant_id=bundle.tenant_id
|
||||
AND routing.public_id::text=bundle.payload->>'routing_version_id'
|
||||
AND routing.config_kind='ROUTING'
|
||||
AND routing.status IN ('PUBLISHED','RETIRED')
|
||||
WHERE rr.tenant_id=$1::uuid
|
||||
AND rr.status='ACTIVE'
|
||||
AND bundle.config_kind='AGENT_BUNDLE'
|
||||
AND bundle.status='PUBLISHED'
|
||||
AND (
|
||||
rr.scope_type='TENANT'
|
||||
OR (rr.scope_type='SUBJECT' AND rr.subject_id=$2::uuid)
|
||||
OR (
|
||||
rr.scope_type='DEVICE_GROUP'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM ops_device_group_member dgm
|
||||
JOIN ops_device_group dg ON dg.id=dgm.group_id AND dg.tenant_id=dgm.tenant_id
|
||||
WHERE dgm.group_id=rr.device_group_id AND dgm.tenant_id=rr.tenant_id
|
||||
AND dgm.device_id=$3::uuid AND dg.status='ACTIVE'
|
||||
)
|
||||
)
|
||||
)
|
||||
AND (
|
||||
rr.rollout_percent=100
|
||||
OR mod((hashtextextended($4,rr.cohort_seed::bigint) & 9223372036854775807),100) < rr.rollout_percent
|
||||
)
|
||||
AND (
|
||||
SELECT count(*) FROM conversation active_conversation
|
||||
JOIN context_snapshot active_snapshot ON active_snapshot.id=active_conversation.context_snapshot_id
|
||||
WHERE active_conversation.tenant_id=rr.tenant_id
|
||||
AND active_conversation.status IN ('PREPARING','ROOM_ISSUED','STARTING_AGENT','ACTIVE','STOPPING')
|
||||
AND active_snapshot.prompt_bundle_version='cfg:' || bundle.public_id::text
|
||||
) < rr.max_concurrent_calls
|
||||
ORDER BY CASE rr.scope_type WHEN 'SUBJECT' THEN 1 WHEN 'DEVICE_GROUP' THEN 2 ELSE 3 END,
|
||||
rr.activated_at DESC,rr.id DESC
|
||||
LIMIT 1`, tenantID, subjectID, deviceID, devicePublicID).Scan(&configID, &engineMode)
|
||||
if err == nil {
|
||||
return configID, engineMode, nil
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", "", err
|
||||
}
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT routing.public_id::text,routing.payload->>'default_engine_mode'
|
||||
FROM runtime_config_rollout rr
|
||||
JOIN runtime_config_version routing
|
||||
ON routing.id=rr.config_version_id AND routing.tenant_id=rr.tenant_id
|
||||
WHERE rr.tenant_id=$1::uuid
|
||||
AND rr.status='ACTIVE'
|
||||
AND routing.config_kind='ROUTING'
|
||||
AND routing.status='PUBLISHED'
|
||||
AND (
|
||||
rr.scope_type='TENANT'
|
||||
OR (rr.scope_type='SUBJECT' AND rr.subject_id=$2::uuid)
|
||||
OR (
|
||||
rr.scope_type='DEVICE_GROUP'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM ops_device_group_member dgm
|
||||
JOIN ops_device_group dg ON dg.id=dgm.group_id AND dg.tenant_id=dgm.tenant_id
|
||||
WHERE dgm.group_id=rr.device_group_id AND dgm.tenant_id=rr.tenant_id
|
||||
AND dgm.device_id=$3::uuid AND dg.status='ACTIVE'
|
||||
)
|
||||
)
|
||||
)
|
||||
AND (
|
||||
rr.rollout_percent=100
|
||||
OR mod((hashtextextended($4,rr.cohort_seed::bigint) & 9223372036854775807),100) < rr.rollout_percent
|
||||
)
|
||||
AND (
|
||||
SELECT count(*) FROM conversation active_conversation
|
||||
JOIN context_snapshot active_snapshot ON active_snapshot.id=active_conversation.context_snapshot_id
|
||||
WHERE active_conversation.tenant_id=rr.tenant_id
|
||||
AND active_conversation.status IN ('PREPARING','ROOM_ISSUED','STARTING_AGENT','ACTIVE','STOPPING')
|
||||
AND active_snapshot.prompt_bundle_version='cfg:' || routing.public_id::text
|
||||
) < rr.max_concurrent_calls
|
||||
ORDER BY CASE rr.scope_type WHEN 'SUBJECT' THEN 1 WHEN 'DEVICE_GROUP' THEN 2 ELSE 3 END,
|
||||
rr.activated_at DESC,rr.id DESC
|
||||
LIMIT 1`, tenantID, subjectID, deviceID, devicePublicID).Scan(&configID, &engineMode)
|
||||
return configID, engineMode, err
|
||||
}
|
||||
|
||||
func (r *Repository) createContextPreparationFailure(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
identity deviceauth.Identity,
|
||||
householdID, subjectID string,
|
||||
profileVersion int64,
|
||||
consentSnapshotID string,
|
||||
memoryVersionsJSON json.RawMessage,
|
||||
promptBundleVersion, engineMode string,
|
||||
privacy Privacy,
|
||||
options AllocationOptions,
|
||||
) (CreateResponse, string, error) {
|
||||
failureValue := struct {
|
||||
FailureCode string `json:"failure_code"`
|
||||
ProfileVersion int64 `json:"profile_version"`
|
||||
PromptVersion string `json:"prompt_bundle_version"`
|
||||
}{contextPreparationFailureCode, profileVersion, promptBundleVersion}
|
||||
failureJSON, _ := json.Marshal(failureValue)
|
||||
digest := sha256.Sum256(failureJSON)
|
||||
contextSnapshotID := uuid.NewString()
|
||||
contextSnapshotPublicID := uuid.NewString()
|
||||
_, err := tx.Exec(ctx, `
|
||||
INSERT INTO context_snapshot (
|
||||
id, public_id, tenant_id, subject_id, profile_version, consent_snapshot_id,
|
||||
memory_versions, prompt_bundle_version, engine_mode, content_digest, sanitized_summary,
|
||||
preparation_status, preparation_failure_code
|
||||
) VALUES ($1::uuid, $2::uuid, $3::uuid, $4::uuid, $5, $6::uuid,
|
||||
$7::jsonb, $8, $9, $10, $11, 'FAILED', $12)`,
|
||||
contextSnapshotID, contextSnapshotPublicID, identity.TenantID, subjectID, profileVersion,
|
||||
consentSnapshotID, memoryVersionsJSON, promptBundleVersion, engineMode,
|
||||
hex.EncodeToString(digest[:]), "背景资料准备失败，未创建房间", contextPreparationFailureCode)
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("create failed context snapshot: %w", err)
|
||||
}
|
||||
|
||||
conversationID := uuid.NewString()
|
||||
conversationPublicID := uuid.NewString()
|
||||
providerTaskRef := "context-failed-" + uuid.NewString()
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO conversation (
|
||||
id, public_id, tenant_id, household_id, subject_id, device_id,
|
||||
consent_snapshot_id, context_snapshot_id, provider_task_ref,
|
||||
engine_mode, retention_mode, status, allocation_expires_at,
|
||||
ended_at, failure_code, created_at, updated_at
|
||||
) VALUES (
|
||||
$1::uuid, $2::uuid, $3::uuid, $4::uuid, $5::uuid, $6::uuid,
|
||||
$7::uuid, $8::uuid, $9, $10, $11, 'FAILED', $12,
|
||||
$13, $14, $13, $13
|
||||
)`, conversationID, conversationPublicID, identity.TenantID, householdID, subjectID, identity.ID,
|
||||
consentSnapshotID, contextSnapshotID, providerTaskRef, engineMode, privacy.RetentionMode,
|
||||
options.AllocationExpires, options.Now, contextPreparationFailureCode)
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("create failed conversation: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (conversation_id, from_status, to_status, reason, source)
|
||||
VALUES
|
||||
($1::uuid, NULL, 'PREPARING', 'create request accepted', 'device'),
|
||||
($1::uuid, 'PREPARING', 'FAILED', 'authorized subject context preparation failed', 'platform')`, conversationID)
|
||||
if err != nil {
|
||||
return CreateResponse{}, "", fmt.Errorf("record context preparation failure: %w", err)
|
||||
}
|
||||
return CreateResponse{
|
||||
ConversationID: conversationPublicID,
|
||||
Status: conversation.StatusFailed,
|
||||
FailureCode: contextPreparationFailureCode,
|
||||
Privacy: privacy,
|
||||
Session: Session{
|
||||
EngineMode: engineMode, HeartbeatIntervalSecond: options.HeartbeatInterval,
|
||||
AllocationExpiresAt: options.AllocationExpires,
|
||||
},
|
||||
}, conversationID, nil
|
||||
}
|
||||
|
||||
func retrievalPolicy(policy string) string {
|
||||
if policy == "" {
|
||||
return "pgvector-normal-v1"
|
||||
}
|
||||
return policy
|
||||
}
|
||||
|
||||
func truncateContextText(value string, limit int) string {
|
||||
runes := []rune(strings.TrimSpace(value))
|
||||
if len(runes) <= limit {
|
||||
return string(runes)
|
||||
}
|
||||
return string(runes[:limit]) + "…"
|
||||
}
|
||||
|
||||
type authorizedSubjectContext struct {
|
||||
Background string `json:"background,omitempty"`
|
||||
Interests []string `json:"interests,omitempty"`
|
||||
ConversationGoal string `json:"conversation_goal,omitempty"`
|
||||
Pace string `json:"pace,omitempty"`
|
||||
PreferredTopics []string `json:"preferred_topics,omitempty"`
|
||||
}
|
||||
|
||||
func authorizedProfileContext(profileDataJSON, preferencesJSON json.RawMessage) (authorizedSubjectContext, error) {
|
||||
var profileData, preferences map[string]any
|
||||
if err := json.Unmarshal(profileDataJSON, &profileData); err != nil {
|
||||
return authorizedSubjectContext{}, fmt.Errorf("decode profile data: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(preferencesJSON, &preferences); err != nil {
|
||||
return authorizedSubjectContext{}, fmt.Errorf("decode conversation preferences: %w", err)
|
||||
}
|
||||
background, err := strictContextString(profileData, "background", 500)
|
||||
if err != nil {
|
||||
return authorizedSubjectContext{}, err
|
||||
}
|
||||
interests, err := strictContextStringList(profileData, "interests", 8, 50)
|
||||
if err != nil {
|
||||
return authorizedSubjectContext{}, err
|
||||
}
|
||||
goal, err := strictContextString(preferences, "conversation_goal", 200)
|
||||
if err != nil {
|
||||
return authorizedSubjectContext{}, err
|
||||
}
|
||||
pace, err := strictContextString(preferences, "pace", 24)
|
||||
if err != nil {
|
||||
return authorizedSubjectContext{}, err
|
||||
}
|
||||
preferredTopics, err := strictContextStringList(preferences, "preferred_topics", 8, 50)
|
||||
if err != nil {
|
||||
return authorizedSubjectContext{}, err
|
||||
}
|
||||
if pace != "slow" && pace != "normal" && pace != "lively" {
|
||||
pace = ""
|
||||
}
|
||||
return authorizedSubjectContext{
|
||||
Background: background,
|
||||
Interests: interests,
|
||||
ConversationGoal: goal,
|
||||
Pace: pace,
|
||||
PreferredTopics: preferredTopics,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildContextSummary(preferredName string, profile authorizedSubjectContext) string {
|
||||
parts := []string{"称呼：" + truncateContextText(preferredName, 120)}
|
||||
if profile.Background != "" {
|
||||
parts = append(parts, "背景资料："+profile.Background)
|
||||
}
|
||||
if len(profile.Interests) > 0 {
|
||||
parts = append(parts, "兴趣："+strings.Join(profile.Interests, "、"))
|
||||
}
|
||||
goal := profile.ConversationGoal
|
||||
if goal == "" {
|
||||
goal = "进行一次自然、尊重的陪伴交流"
|
||||
}
|
||||
parts = append(parts, "本次交流目标："+goal)
|
||||
if profile.Pace != "" {
|
||||
paceLabels := map[string]string{"slow": "放慢语速并留出回应时间", "normal": "自然语速", "lively": "轻快但保持清晰"}
|
||||
parts = append(parts, "交流节奏："+paceLabels[profile.Pace])
|
||||
}
|
||||
if len(profile.PreferredTopics) > 0 {
|
||||
parts = append(parts, "优先话题："+strings.Join(profile.PreferredTopics, "、"))
|
||||
}
|
||||
return strings.Join(parts, "；")
|
||||
}
|
||||
|
||||
func strictContextString(values map[string]any, key string, limit int) (string, error) {
|
||||
raw, exists := values[key]
|
||||
if !exists || raw == nil {
|
||||
return "", nil
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s must be a string", key)
|
||||
}
|
||||
return truncateContextText(value, limit), nil
|
||||
}
|
||||
|
||||
func strictContextStringList(values map[string]any, key string, maxItems, itemLimit int) ([]string, error) {
|
||||
raw, exists := values[key]
|
||||
if !exists || raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rawItems, ok := raw.([]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s must be an array of strings", key)
|
||||
}
|
||||
items := make([]string, 0, min(len(rawItems), maxItems))
|
||||
for _, raw := range rawItems {
|
||||
value, ok := raw.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s must contain only strings", key)
|
||||
}
|
||||
value = truncateContextText(value, itemLimit)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, value)
|
||||
if len(items) == maxItems {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func privacyFromConsent(consents json.RawMessage) Privacy {
|
||||
values := map[string]bool{}
|
||||
_ = json.Unmarshal(consents, &values)
|
||||
privacy := Privacy{
|
||||
RecordNear: values["raw_audio_storage"] || values["audio_recording"],
|
||||
RecordFar: values["raw_audio_storage"] || values["audio_recording"],
|
||||
UploadAllowed: values["raw_audio_storage"] || values["audio_recording"], RetentionMode: "none",
|
||||
}
|
||||
if values["transcript_storage"] || values["transcript"] {
|
||||
privacy.RetentionMode = "transcript_only"
|
||||
}
|
||||
if values["raw_audio_storage"] || values["audio_recording"] {
|
||||
privacy.RetentionMode = "transcript_and_audio"
|
||||
}
|
||||
return privacy
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/deviceauth"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func TestRepositoryCreatesOnlyRoomIssuedStateBeforeActivation(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "orchestration-repository-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
tx, err := pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)
|
||||
identity := deviceauth.Identity{
|
||||
ID: "10000000-0000-0000-0000-000000000501", PublicID: "10000000-0000-0000-0000-000000000511",
|
||||
TenantID: "10000000-0000-0000-0000-000000000001", Status: "registered",
|
||||
}
|
||||
resume := true
|
||||
if _, err := tx.Exec(ctx, `UPDATE device SET minimum_firmware_version='9.0.0' WHERE id=$1::uuid`, identity.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, firmwareErr := NewRepository().Create(ctx, tx, identity, CreateInput{RequestedGoal: "open_conversation"}, AllocationOptions{
|
||||
Now: now, AllocationExpires: now.Add(30 * time.Second), TokenExpires: now.Add(5 * time.Minute),
|
||||
RTCAppID: "test-app", EngineMode: "s2s_hybrid", HeartbeatInterval: 15,
|
||||
}); !errors.Is(firmwareErr, ErrFirmwareUpdateRequired) {
|
||||
t.Fatalf("minimum firmware error=%v", firmwareErr)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE device SET minimum_firmware_version='0.1.0-sim' WHERE id=$1::uuid`, identity.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, strictErr := NewRepository().Create(ctx, tx, identity, CreateInput{RequestedGoal: "open_conversation"}, AllocationOptions{
|
||||
Now: now, AllocationExpires: now.Add(30 * time.Second), TokenExpires: now.Add(5 * time.Minute),
|
||||
RTCAppID: "test-app", EngineMode: "s2s_hybrid", HeartbeatInterval: 15, RequirePublishedRouting: true,
|
||||
}); !errors.Is(strictErr, ErrRuntimeConfigUnavailable) {
|
||||
t.Fatalf("strict production routing error=%v", strictErr)
|
||||
}
|
||||
response, internalID, err := NewRepository().Create(ctx, tx, identity, CreateInput{
|
||||
DeviceTime: now, Capabilities: Capabilities{RTC: true, AudioTracks: []string{"near", "far"}, UploadResume: &resume},
|
||||
Network: Network{Type: "test"}, RequestedGoal: "open_conversation",
|
||||
}, AllocationOptions{
|
||||
Now: now, AllocationExpires: now.Add(30 * time.Second), TokenExpires: now.Add(5 * time.Minute),
|
||||
RTCAppID: "test-app", EngineMode: "s2s_hybrid", HeartbeatInterval: 15,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Status != "ROOM_ISSUED" || response.RTC.Token != "" || response.Privacy.RetentionMode != "transcript_only" {
|
||||
t.Fatalf("response = %#v", response)
|
||||
}
|
||||
var status, taskStatus string
|
||||
var credentialIssuedAt, credentialExpiresAt time.Time
|
||||
var historyCount int
|
||||
if err := tx.QueryRow(ctx, "SELECT status FROM conversation WHERE id=$1::uuid", internalID).Scan(&status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, "SELECT status,credential_issued_at,credential_expires_at FROM rtc_task WHERE conversation_id=$1::uuid", internalID).Scan(&taskStatus, &credentialIssuedAt, &credentialExpiresAt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, "SELECT count(*) FROM conversation_state_history WHERE conversation_id=$1::uuid", internalID).Scan(&historyCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "ROOM_ISSUED" || taskStatus != "ALLOCATED" || historyCount != 2 {
|
||||
t.Fatalf("status=%s task=%s history=%d", status, taskStatus, historyCount)
|
||||
}
|
||||
if !credentialIssuedAt.Equal(now) || !credentialExpiresAt.Equal(now.Add(5*time.Minute)) {
|
||||
t.Fatalf("credential metadata issued=%s expires=%s", credentialIssuedAt, credentialExpiresAt)
|
||||
}
|
||||
activation, err := NewRepository().Activate(ctx, tx, identity, response.ConversationID, "request-activate", ActivateInput{
|
||||
JoinedAt: now.Add(time.Second), RoomID: response.RTC.RoomID, RTCUserID: response.RTC.UserID,
|
||||
RTCJoinResult: "success", ConfigRevision: 1,
|
||||
}, now.Add(2*time.Second))
|
||||
if err != nil || !activation.StartRequested || activation.Response.Status != "STARTING_AGENT" {
|
||||
t.Fatalf("activation=%#v err=%v", activation, err)
|
||||
}
|
||||
duplicate, err := NewRepository().Activate(ctx, tx, identity, response.ConversationID, "request-activate-duplicate", ActivateInput{
|
||||
JoinedAt: now.Add(time.Second), RoomID: response.RTC.RoomID, RTCUserID: response.RTC.UserID,
|
||||
RTCJoinResult: "success", ConfigRevision: 1,
|
||||
}, now.Add(3*time.Second))
|
||||
if err != nil || duplicate.StartRequested || duplicate.Response.Status != "STARTING_AGENT" {
|
||||
t.Fatalf("duplicate activation=%#v err=%v", duplicate, err)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, "SELECT status FROM rtc_task WHERE conversation_id=$1::uuid", internalID).Scan(&taskStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, "SELECT count(*) FROM conversation_state_history WHERE conversation_id=$1::uuid", internalID).Scan(&historyCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if taskStatus != "STARTING" || historyCount != 3 {
|
||||
t.Fatalf("activated task=%s history=%d", taskStatus, historyCount)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "UPDATE conversation SET status='ACTIVE' WHERE id=$1::uuid", internalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "UPDATE rtc_task SET status='ACTIVE' WHERE conversation_id=$1::uuid", internalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
monotonicMS := int64(9000)
|
||||
heartbeatInput := HeartbeatInput{
|
||||
SequenceNo: 2, DeviceTime: now.Add(4 * time.Second), MonotonicMS: &monotonicMS,
|
||||
ConversationStatus: "ACTIVE", RTC: RTCHeartbeat{Joined: true, Uplink: true, Downlink: true},
|
||||
Network: Network{Type: "wifi"}, FirmwareVersion: "1.0.0", ReportedConfigRevision: 1,
|
||||
RecordingQueue: RecordingQueue{PendingSegments: 2}, CommandResults: []json.RawMessage{},
|
||||
}
|
||||
heartbeat, err := NewRepository().Heartbeat(ctx, tx, identity, response.ConversationID, heartbeatInput, now.Add(4*time.Second), time.Minute, 5*time.Minute, time.Minute)
|
||||
if err != nil || !heartbeat.Updated || heartbeat.Response.LeaseExpiresAt == nil || heartbeat.Response.Actions[0].Type != "continue" {
|
||||
t.Fatalf("heartbeat=%#v err=%v", heartbeat, err)
|
||||
}
|
||||
originalLease := *heartbeat.Response.LeaseExpiresAt
|
||||
heartbeatInput.SequenceNo = 1
|
||||
stale, err := NewRepository().Heartbeat(ctx, tx, identity, response.ConversationID, heartbeatInput, now.Add(30*time.Second), time.Minute, 5*time.Minute, time.Minute)
|
||||
if err != nil || stale.Updated || stale.Response.LeaseExpiresAt == nil || !stale.Response.LeaseExpiresAt.Equal(originalLease) {
|
||||
t.Fatalf("stale heartbeat=%#v err=%v", stale, err)
|
||||
}
|
||||
heartbeatInput.SequenceNo = 3
|
||||
heartbeatInput.ReportedConfigRevision = 2
|
||||
if _, err := NewRepository().Heartbeat(ctx, tx, identity, response.ConversationID, heartbeatInput, now.Add(31*time.Second), time.Minute, 5*time.Minute, time.Minute); !errors.Is(err, ErrConfigVersionConflict) {
|
||||
t.Fatalf("future configuration revision error=%v", err)
|
||||
}
|
||||
var storedSequence int64
|
||||
if err := tx.QueryRow(ctx, "SELECT last_heartbeat_seq FROM conversation WHERE id=$1::uuid", internalID).Scan(&storedSequence); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if storedSequence != 2 {
|
||||
t.Fatalf("last heartbeat sequence=%d, want 2", storedSequence)
|
||||
}
|
||||
if _, _, err := NewRepository().Create(ctx, tx, identity, CreateInput{RequestedGoal: "open_conversation"}, AllocationOptions{
|
||||
Now: now, AllocationExpires: now.Add(time.Minute), TokenExpires: now.Add(time.Minute), RTCAppID: "test-app", EngineMode: "s2s_hybrid", HeartbeatInterval: 15,
|
||||
}); !errors.Is(err, ErrConversationAlreadyActive) {
|
||||
t.Fatalf("second active conversation error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/conversation"
|
||||
"data-terminal-cloud/internal/deviceauth"
|
||||
"data-terminal-cloud/internal/embeddingprovider"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
"data-terminal-cloud/internal/rtc"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type ServiceOptions struct {
|
||||
RTCAppID string
|
||||
VoiceProvider string
|
||||
EngineMode string
|
||||
AllocationTTL time.Duration
|
||||
TokenTTL time.Duration
|
||||
HeartbeatInterval int
|
||||
RequirePublishedRouting bool
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
executor *idempotency.Executor
|
||||
repository *Repository
|
||||
audit *audit.Repository
|
||||
outbox *outbox.Store
|
||||
tokens rtc.TokenIssuer
|
||||
leases LeaseCache
|
||||
options ServiceOptions
|
||||
pool *pgxpool.Pool
|
||||
embedder embeddingprovider.Provider
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func (s *Service) ConfigureMemoryEmbedding(pool *pgxpool.Pool, provider embeddingprovider.Provider) error {
|
||||
if pool == nil || provider == nil {
|
||||
return errors.New("memory embedding pool and provider are required")
|
||||
}
|
||||
s.pool = pool
|
||||
s.embedder = provider
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewService(
|
||||
executor *idempotency.Executor,
|
||||
repository *Repository,
|
||||
auditRepository *audit.Repository,
|
||||
outboxStore *outbox.Store,
|
||||
tokens rtc.TokenIssuer,
|
||||
leases LeaseCache,
|
||||
options ServiceOptions,
|
||||
) (*Service, error) {
|
||||
if executor == nil || repository == nil || auditRepository == nil || outboxStore == nil || tokens == nil || leases == nil {
|
||||
return nil, errors.New("orchestration dependencies are required")
|
||||
}
|
||||
if options.RTCAppID == "" || options.EngineMode == "" || options.AllocationTTL <= 0 || options.TokenTTL <= 0 || options.HeartbeatInterval <= 0 {
|
||||
return nil, errors.New("valid orchestration options are required")
|
||||
}
|
||||
if options.VoiceProvider == "" {
|
||||
options.VoiceProvider = "mock"
|
||||
}
|
||||
return &Service{executor: executor, repository: repository, audit: auditRepository, outbox: outboxStore,
|
||||
tokens: tokens, leases: leases, options: options, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (s *Service) Create(
|
||||
ctx context.Context,
|
||||
identity deviceauth.Identity,
|
||||
idempotencyKey, requestID string,
|
||||
rawRequest []byte,
|
||||
input CreateInput,
|
||||
) (CreateResponse, bool, error) {
|
||||
tenantID := identity.TenantID
|
||||
request := idempotency.Request{
|
||||
TenantID: &tenantID, ActorKey: "device:" + identity.PublicID,
|
||||
Scope: "POST /device/v2/conversations", Key: idempotencyKey,
|
||||
RequestHash: idempotency.Hash(rawRequest), ExpiresAfter: 24 * time.Hour,
|
||||
}
|
||||
if cached, found, lookupErr := s.executor.Lookup(ctx, request); lookupErr != nil {
|
||||
return CreateResponse{}, false, lookupErr
|
||||
} else if found {
|
||||
return s.finalizeCreateResult(cached)
|
||||
}
|
||||
memoryVector, memoryPolicy, err := s.prepareMemoryQuery(ctx, identity, input.RequestedGoal)
|
||||
if err != nil {
|
||||
return CreateResponse{}, false, err
|
||||
}
|
||||
result, err := s.executor.Execute(ctx, request, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
now := s.now().UTC()
|
||||
response, internalID, createErr := s.repository.Create(operationContext, tx, identity, input, AllocationOptions{
|
||||
Now: now, AllocationExpires: now.Add(s.options.AllocationTTL), TokenExpires: now.Add(s.options.TokenTTL),
|
||||
RTCAppID: s.options.RTCAppID, VoiceProvider: s.options.VoiceProvider, EngineMode: s.options.EngineMode, HeartbeatInterval: s.options.HeartbeatInterval,
|
||||
MemoryQueryVector: memoryVector, MemoryPolicy: memoryPolicy, RequirePublishedRouting: s.options.RequirePublishedRouting,
|
||||
})
|
||||
if createErr != nil {
|
||||
return idempotency.Result{}, createErr
|
||||
}
|
||||
body, marshalErr := json.Marshal(response)
|
||||
if marshalErr != nil {
|
||||
return idempotency.Result{}, fmt.Errorf("encode create conversation result: %w", marshalErr)
|
||||
}
|
||||
actorPublicID := identity.PublicID
|
||||
auditResult := "success"
|
||||
if response.Status == conversation.StatusFailed {
|
||||
auditResult = "failure"
|
||||
}
|
||||
if auditErr := s.audit.Record(operationContext, tx, audit.Event{
|
||||
TenantID: &tenantID, ActorType: "device", ActorPublicID: &actorPublicID,
|
||||
Action: "conversation.create", ResourceType: "conversation", ResourceID: &internalID,
|
||||
RequestID: requestID, Purpose: input.RequestedGoal, Result: auditResult,
|
||||
Detail: map[string]any{"status": response.Status, "failure_code": response.FailureCode},
|
||||
}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
statusCode := http.StatusCreated
|
||||
if response.Status == conversation.StatusFailed {
|
||||
statusCode = http.StatusUnprocessableEntity
|
||||
} else {
|
||||
deduplicationKey := response.ConversationID
|
||||
if _, outboxErr := s.outbox.Enqueue(operationContext, tx, outbox.NewEvent{
|
||||
TenantID: &tenantID, AggregateType: "conversation", AggregateID: internalID,
|
||||
EventType: "conversation.room_issued", DeduplicationKey: &deduplicationKey,
|
||||
Payload: json.RawMessage(fmt.Sprintf(`{"conversation_id":%q,"status":"ROOM_ISSUED"}`, response.ConversationID)),
|
||||
}); outboxErr != nil {
|
||||
return idempotency.Result{}, outboxErr
|
||||
}
|
||||
}
|
||||
return idempotency.Result{StatusCode: statusCode, Body: body, ResourceID: &internalID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return CreateResponse{}, false, err
|
||||
}
|
||||
return s.finalizeCreateResult(result)
|
||||
}
|
||||
|
||||
func (s *Service) finalizeCreateResult(result idempotency.Result) (CreateResponse, bool, error) {
|
||||
var response CreateResponse
|
||||
if err := json.Unmarshal(result.Body, &response); err != nil {
|
||||
return CreateResponse{}, result.Replayed, fmt.Errorf("decode stored create conversation result: %w", err)
|
||||
}
|
||||
if response.Status == conversation.StatusFailed {
|
||||
return CreateResponse{}, result.Replayed, &CreateFailureError{
|
||||
ConversationID: response.ConversationID,
|
||||
FailureCode: response.FailureCode,
|
||||
}
|
||||
}
|
||||
token, err := s.tokens.Issue(response.RTC.RoomID, response.RTC.UserID, response.RTC.ExpiresAt)
|
||||
if err != nil {
|
||||
return CreateResponse{}, result.Replayed, fmt.Errorf("issue RTC token: %w", err)
|
||||
}
|
||||
response.RTC.Token = token
|
||||
return response, result.Replayed, nil
|
||||
}
|
||||
|
||||
func (s *Service) prepareMemoryQuery(ctx context.Context, identity deviceauth.Identity, requestedGoal string) (string, string, error) {
|
||||
if s.pool == nil || s.embedder == nil {
|
||||
return "", "", nil
|
||||
}
|
||||
var subjectName string
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT sp.preferred_name
|
||||
FROM device d
|
||||
JOIN device_binding b ON b.device_id=d.id AND b.tenant_id=d.tenant_id AND b.unbound_at IS NULL
|
||||
JOIN subject_profile sp ON sp.id=b.subject_id AND sp.tenant_id=b.tenant_id AND sp.status='active'
|
||||
WHERE d.id=$1::uuid AND d.tenant_id=$2::uuid AND d.status='registered'
|
||||
AND EXISTS(SELECT 1 FROM consent_snapshot cs WHERE cs.tenant_id=d.tenant_id AND cs.subject_id=sp.id
|
||||
AND cs.effective_consents @> '{"voice_chat":true}'::jsonb)
|
||||
AND EXISTS(SELECT 1 FROM consent_record cr WHERE cr.tenant_id=d.tenant_id AND cr.subject_id=sp.id
|
||||
AND cr.consent_type='personalized_memory' AND cr.withdrawn_at IS NULL
|
||||
AND (cr.expires_at IS NULL OR cr.expires_at>now()))`, identity.ID, identity.TenantID).Scan(&subjectName)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", "pgvector-not-eligible-v1", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("prepare memory retrieval query: %w", err)
|
||||
}
|
||||
result, err := s.embedder.Embed(ctx, strings.TrimSpace(subjectName+" "+requestedGoal))
|
||||
if err != nil {
|
||||
if outbox.IsRetryable(err) {
|
||||
return "", "pgvector-degraded-v1", nil
|
||||
}
|
||||
return "", "", fmt.Errorf("generate memory retrieval embedding: %w", err)
|
||||
}
|
||||
literal, err := embeddingprovider.Literal(result.Vector)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("encode memory retrieval embedding: %w", err)
|
||||
}
|
||||
return literal, "pgvector-" + result.ProviderRef + "-normal-v1", nil
|
||||
}
|
||||
|
||||
func (s *Service) Activate(
|
||||
ctx context.Context,
|
||||
identity deviceauth.Identity,
|
||||
conversationID, idempotencyKey, requestID string,
|
||||
rawRequest []byte,
|
||||
input ActivateInput,
|
||||
) (ActivateResponse, bool, error) {
|
||||
tenantID := identity.TenantID
|
||||
scope := "POST /device/v2/conversations/" + conversationID + "/activate"
|
||||
result, err := s.executor.Execute(ctx, idempotency.Request{
|
||||
TenantID: &tenantID, ActorKey: "device:" + identity.PublicID, Scope: scope,
|
||||
Key: idempotencyKey, RequestHash: idempotency.Hash(rawRequest), ExpiresAfter: 24 * time.Hour,
|
||||
}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
activation, activateErr := s.repository.Activate(
|
||||
operationContext, tx, identity, conversationID, requestID, input, s.now().UTC(),
|
||||
)
|
||||
if activateErr != nil {
|
||||
return idempotency.Result{}, activateErr
|
||||
}
|
||||
if activation.StartRequested {
|
||||
deduplicationKey := activation.ProviderTaskRef
|
||||
payload, _ := json.Marshal(map[string]string{
|
||||
"conversation_id": conversationID, "provider_task_ref": activation.ProviderTaskRef,
|
||||
})
|
||||
if _, enqueueErr := s.outbox.Enqueue(operationContext, tx, outbox.NewEvent{
|
||||
TenantID: &tenantID, AggregateType: "conversation", AggregateID: activation.InternalID,
|
||||
EventType: "conversation.agent_start_requested", DeduplicationKey: &deduplicationKey,
|
||||
Payload: payload,
|
||||
}); enqueueErr != nil {
|
||||
return idempotency.Result{}, enqueueErr
|
||||
}
|
||||
}
|
||||
actorPublicID := identity.PublicID
|
||||
if auditErr := s.audit.Record(operationContext, tx, audit.Event{
|
||||
TenantID: &tenantID, ActorType: "device", ActorPublicID: &actorPublicID,
|
||||
Action: "conversation.activate", ResourceType: "conversation", ResourceID: &activation.InternalID,
|
||||
RequestID: requestID, Purpose: "confirm_rtc_join", Result: "success",
|
||||
Detail: map[string]any{"status": activation.Response.Status, "start_requested": activation.StartRequested},
|
||||
}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
body, marshalErr := json.Marshal(activation.Response)
|
||||
if marshalErr != nil {
|
||||
return idempotency.Result{}, marshalErr
|
||||
}
|
||||
return idempotency.Result{StatusCode: http.StatusAccepted, Body: body, ResourceID: &activation.InternalID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return ActivateResponse{}, false, err
|
||||
}
|
||||
var response ActivateResponse
|
||||
if err := json.Unmarshal(result.Body, &response); err != nil {
|
||||
return ActivateResponse{}, result.Replayed, fmt.Errorf("decode stored activation result: %w", err)
|
||||
}
|
||||
return response, result.Replayed, nil
|
||||
}
|
||||
|
||||
func (s *Service) GetDeviceState(ctx context.Context, identity deviceauth.Identity, conversationID string) (DeviceConversationState, error) {
|
||||
if s.pool == nil {
|
||||
return DeviceConversationState{}, errors.New("conversation state database is not configured")
|
||||
}
|
||||
var state DeviceConversationState
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT c.public_id::text,c.status,COALESCE(c.failure_code,'')
|
||||
FROM conversation c
|
||||
WHERE c.public_id=$1::uuid AND c.device_id=$2::uuid AND c.tenant_id=$3::uuid`,
|
||||
conversationID, identity.ID, identity.TenantID,
|
||||
).Scan(&state.ConversationID, &state.Status, &state.FailureCode)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return DeviceConversationState{}, ErrConversationNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return DeviceConversationState{}, fmt.Errorf("read device conversation state: %w", err)
|
||||
}
|
||||
switch state.Status {
|
||||
case conversation.StatusActive:
|
||||
state.Action = "continue"
|
||||
case conversation.StatusStopping, conversation.StatusCompleted, conversation.StatusFailed, conversation.StatusExpired:
|
||||
state.Action = "leave_room"
|
||||
default:
|
||||
state.Action = "wait"
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func (s *Service) Stop(
|
||||
ctx context.Context,
|
||||
identity deviceauth.Identity,
|
||||
conversationID, idempotencyKey, requestID string,
|
||||
rawRequest []byte,
|
||||
input StopInput,
|
||||
) (StopResponse, bool, error) {
|
||||
tenantID := identity.TenantID
|
||||
scope := "POST /device/v2/conversations/" + conversationID + "/stop"
|
||||
result, err := s.executor.Execute(ctx, idempotency.Request{
|
||||
TenantID: &tenantID, ActorKey: "device:" + identity.PublicID, Scope: scope,
|
||||
Key: idempotencyKey, RequestHash: idempotency.Hash(rawRequest), ExpiresAfter: 24 * time.Hour,
|
||||
}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
stop, stopErr := s.repository.Stop(operationContext, tx, identity, conversationID, requestID, input, s.now().UTC())
|
||||
if stopErr != nil {
|
||||
return idempotency.Result{}, stopErr
|
||||
}
|
||||
if stop.StopRequested {
|
||||
deduplicationKey := stop.ProviderTaskRef
|
||||
payload, _ := json.Marshal(map[string]string{
|
||||
"conversation_id": conversationID, "provider_task_ref": stop.ProviderTaskRef, "reason": input.Reason,
|
||||
})
|
||||
if _, enqueueErr := s.outbox.Enqueue(operationContext, tx, outbox.NewEvent{
|
||||
TenantID: &tenantID, AggregateType: "conversation", AggregateID: stop.InternalID,
|
||||
EventType: "conversation.agent_stop_requested", DeduplicationKey: &deduplicationKey,
|
||||
Payload: payload,
|
||||
}); enqueueErr != nil {
|
||||
return idempotency.Result{}, enqueueErr
|
||||
}
|
||||
}
|
||||
actorPublicID := identity.PublicID
|
||||
if auditErr := s.audit.Record(operationContext, tx, audit.Event{
|
||||
TenantID: &tenantID, ActorType: "device", ActorPublicID: &actorPublicID,
|
||||
Action: "conversation.stop", ResourceType: "conversation", ResourceID: &stop.InternalID,
|
||||
RequestID: requestID, Purpose: input.Reason, Result: "success",
|
||||
Detail: map[string]any{"status": stop.Response.Status, "stop_requested": stop.StopRequested},
|
||||
}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
body, marshalErr := json.Marshal(stop.Response)
|
||||
if marshalErr != nil {
|
||||
return idempotency.Result{}, marshalErr
|
||||
}
|
||||
return idempotency.Result{StatusCode: http.StatusAccepted, Body: body, ResourceID: &stop.InternalID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return StopResponse{}, false, err
|
||||
}
|
||||
var response StopResponse
|
||||
if err := json.Unmarshal(result.Body, &response); err != nil {
|
||||
return StopResponse{}, result.Replayed, fmt.Errorf("decode stored stop result: %w", err)
|
||||
}
|
||||
_ = s.leases.End(ctx, conversationID, identity.PublicID, s.now().UTC())
|
||||
return response, result.Replayed, nil
|
||||
}
|
||||
@@ -0,0 +1,890 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/deviceauth"
|
||||
"data-terminal-cloud/internal/embeddingprovider"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
platformidentity "data-terminal-cloud/internal/identity"
|
||||
"data-terminal-cloud/internal/opsconfig"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
"data-terminal-cloud/internal/ratelimit"
|
||||
"data-terminal-cloud/internal/rtc"
|
||||
"data-terminal-cloud/internal/voiceprovider"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const (
|
||||
testDeviceID = "90000000-0000-0000-0000-000000000501"
|
||||
testDevicePublicID = "90000000-0000-0000-0000-000000000511"
|
||||
testSubjectID = "90000000-0000-0000-0000-000000000401"
|
||||
testSubjectPublicID = "90000000-0000-0000-0000-000000000411"
|
||||
testPriceBookID = "90000000-0000-0000-0000-000000000601"
|
||||
testPersonaConfigID = "90000000-0000-0000-0000-000000000701"
|
||||
testPersonaConfigPublicID = "90000000-0000-0000-0000-000000000711"
|
||||
testVoiceConfigID = "90000000-0000-0000-0000-000000000702"
|
||||
testVoiceConfigPublicID = "90000000-0000-0000-0000-000000000712"
|
||||
testModelConfigID = "90000000-0000-0000-0000-000000000703"
|
||||
testModelConfigPublicID = "90000000-0000-0000-0000-000000000713"
|
||||
testRoutingConfigID = "90000000-0000-0000-0000-000000000704"
|
||||
testRoutingConfigPublicID = "90000000-0000-0000-0000-000000000714"
|
||||
testBundleConfigKey = "orchestration-test-agent-bundle"
|
||||
testBundleCreateKey = "orchestration-agent-bundle-create"
|
||||
testBundleEvaluationKey = "orchestration-agent-bundle-evaluation"
|
||||
)
|
||||
|
||||
type testLeaseCache struct{}
|
||||
|
||||
func (testLeaseCache) Refresh(context.Context, LeaseRecord) error { return nil }
|
||||
func (testLeaseCache) End(context.Context, string, string, time.Time) error { return nil }
|
||||
|
||||
type failingTokenIssuer struct{}
|
||||
|
||||
func (failingTokenIssuer) Issue(string, string, time.Time) (string, error) {
|
||||
return "", errors.New("simulated token signing failure")
|
||||
}
|
||||
|
||||
type countingEmbedder struct{ calls atomic.Int32 }
|
||||
|
||||
func (e *countingEmbedder) Embed(ctx context.Context, content string) (embeddingprovider.Result, error) {
|
||||
e.calls.Add(1)
|
||||
return (embeddingprovider.Local{}).Embed(ctx, content)
|
||||
}
|
||||
|
||||
type testVoiceProvider struct {
|
||||
startRequest voiceprovider.StartRequest
|
||||
stopRequest voiceprovider.StopRequest
|
||||
startStatus string
|
||||
stopStatus string
|
||||
queryStatus string
|
||||
startErr error
|
||||
stopErr error
|
||||
queryErr error
|
||||
}
|
||||
|
||||
func (p *testVoiceProvider) Start(_ context.Context, request voiceprovider.StartRequest) (voiceprovider.StartResponse, error) {
|
||||
p.startRequest = request
|
||||
if p.startErr != nil {
|
||||
return voiceprovider.StartResponse{}, p.startErr
|
||||
}
|
||||
status := p.startStatus
|
||||
if status == "" {
|
||||
status = "STARTING"
|
||||
}
|
||||
return voiceprovider.StartResponse{TaskRef: request.TaskRef, Status: status}, nil
|
||||
}
|
||||
|
||||
func (p *testVoiceProvider) Query(_ context.Context, request voiceprovider.QueryRequest) (voiceprovider.QueryResponse, error) {
|
||||
if p.queryErr != nil {
|
||||
return voiceprovider.QueryResponse{}, p.queryErr
|
||||
}
|
||||
status := p.queryStatus
|
||||
if status == "" {
|
||||
status = "STARTING"
|
||||
}
|
||||
return voiceprovider.QueryResponse{TaskRef: request.TaskRef, Status: status}, nil
|
||||
}
|
||||
|
||||
func (p *testVoiceProvider) Stop(_ context.Context, request voiceprovider.StopRequest) (voiceprovider.StopResponse, error) {
|
||||
p.stopRequest = request
|
||||
if p.stopErr != nil {
|
||||
return voiceprovider.StopResponse{}, p.stopErr
|
||||
}
|
||||
status := p.stopStatus
|
||||
if status == "" {
|
||||
status = "STOPPING"
|
||||
}
|
||||
return voiceprovider.StopResponse{TaskRef: request.TaskRef, Status: status}, nil
|
||||
}
|
||||
|
||||
type allowProviderStart struct{}
|
||||
|
||||
func (allowProviderStart) Allow(context.Context, string, string) (ratelimit.Result, error) {
|
||||
return ratelimit.Result{Allowed: true}, nil
|
||||
}
|
||||
|
||||
func TestServiceIdempotentlyCreatesConversationWithoutPersistingToken(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "orchestration-service-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
cleanupOrchestrationFixture(t, ctx, pool)
|
||||
defer cleanupOrchestrationFixture(t, context.Background(), pool)
|
||||
fixtureStatements := []struct {
|
||||
query string
|
||||
args []any
|
||||
}{
|
||||
{`INSERT INTO subject_profile (id, public_id, tenant_id, household_id, preferred_name, profile_data, conversation_preferences, status)
|
||||
VALUES ($1::uuid, '90000000-0000-0000-0000-000000000411', '10000000-0000-0000-0000-000000000001', '10000000-0000-0000-0000-000000000201', '服务测试长者',
|
||||
'{"background":"年轻时从事铁路工作","interests":["老照片","家乡变化"],"internal_note":"不得进入语音上下文"}',
|
||||
'{"conversation_goal":"聊聊第一次参加工作的经历","pace":"slow","preferred_topics":["工作经历"]}', 'active')`, []any{testSubjectID}},
|
||||
{`INSERT INTO device (id, public_id, tenant_id, serial_digest, model, status, credential_ref)
|
||||
VALUES ($1::uuid, $2::uuid, '10000000-0000-0000-0000-000000000001', repeat('9',64), 'service-test', 'registered', 'test://service')`, []any{testDeviceID, testDevicePublicID}},
|
||||
{`INSERT INTO device_binding (tenant_id, device_id, household_id, subject_id, reason)
|
||||
VALUES ('10000000-0000-0000-0000-000000000001', $1::uuid, '10000000-0000-0000-0000-000000000201', $2::uuid, 'integration test')`, []any{testDeviceID, testSubjectID}},
|
||||
{`INSERT INTO device_twin (device_id, tenant_id) VALUES ($1::uuid, '10000000-0000-0000-0000-000000000001')`, []any{testDeviceID}},
|
||||
{`INSERT INTO consent_snapshot (tenant_id, subject_id, snapshot_version, effective_consents)
|
||||
VALUES ('10000000-0000-0000-0000-000000000001', $1::uuid, 1, '{"voice_chat":true,"transcript":true,"audio_recording":false}'::jsonb)`, []any{testSubjectID}},
|
||||
{`INSERT INTO consent_record (tenant_id,subject_id,consent_type,policy_version,notice_digest,actor_user_id)
|
||||
VALUES ('10000000-0000-0000-0000-000000000001',$1::uuid,'personalized_memory','orchestration-service-test',repeat('a',64),'10000000-0000-0000-0000-000000000101')`, []any{testSubjectID}},
|
||||
{`INSERT INTO runtime_config_version (
|
||||
id,public_id,tenant_id,config_key,display_name,config_kind,version_no,status,payload,change_summary,
|
||||
created_by,submitted_by,submitted_at,approved_by,approved_at,published_by,published_at
|
||||
) VALUES (
|
||||
$1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','orchestration-test-persona','编排测试人设','PERSONA',1,'PUBLISHED',
|
||||
'{"system_prompt":"你是已审核发布的陪伴助手。","tone":"warm"}'::jsonb,'编排集成测试',
|
||||
'10000000-0000-0000-0000-000000000102','10000000-0000-0000-0000-000000000102','2026-08-23T11:00:00Z',
|
||||
'10000000-0000-0000-0000-000000000102','2026-08-23T11:01:00Z','10000000-0000-0000-0000-000000000102','2026-08-23T11:02:00Z'
|
||||
)`, []any{testPersonaConfigID, testPersonaConfigPublicID}},
|
||||
{`INSERT INTO runtime_config_version (
|
||||
id,public_id,tenant_id,config_key,display_name,config_kind,version_no,status,payload,change_summary,
|
||||
created_by,submitted_by,submitted_at,approved_by,approved_at,published_by,published_at
|
||||
) VALUES (
|
||||
$1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','orchestration-test-voice','编排测试音色','VOICE',1,'PUBLISHED',
|
||||
'{"provider":"mock","voice_resource_ref":"voice://orchestration/approved"}'::jsonb,'编排集成测试',
|
||||
'10000000-0000-0000-0000-000000000102','10000000-0000-0000-0000-000000000102','2026-08-23T11:00:00Z',
|
||||
'10000000-0000-0000-0000-000000000102','2026-08-23T11:01:00Z','10000000-0000-0000-0000-000000000102','2026-08-23T11:02:00Z'
|
||||
)`, []any{testVoiceConfigID, testVoiceConfigPublicID}},
|
||||
{`INSERT INTO runtime_config_version (
|
||||
id,public_id,tenant_id,config_key,display_name,config_kind,version_no,status,payload,change_summary,
|
||||
created_by,submitted_by,submitted_at,approved_by,approved_at,published_by,published_at
|
||||
) VALUES (
|
||||
$1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','orchestration-test-model','编排测试模型','MODEL',1,'PUBLISHED',
|
||||
'{"provider":"mock","model_id":"model://orchestration/approved"}'::jsonb,'编排集成测试',
|
||||
'10000000-0000-0000-0000-000000000102','10000000-0000-0000-0000-000000000102','2026-08-23T11:00:00Z',
|
||||
'10000000-0000-0000-0000-000000000102','2026-08-23T11:01:00Z','10000000-0000-0000-0000-000000000102','2026-08-23T11:02:00Z'
|
||||
)`, []any{testModelConfigID, testModelConfigPublicID}},
|
||||
{`INSERT INTO runtime_config_version (
|
||||
id,public_id,tenant_id,config_key,display_name,config_kind,version_no,status,payload,change_summary,
|
||||
created_by,submitted_by,submitted_at,approved_by,approved_at,published_by,published_at
|
||||
) VALUES (
|
||||
$1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','orchestration-test-routing','编排测试路由','ROUTING',1,'PUBLISHED',
|
||||
'{"default_engine_mode":"cascade","routes":[{"priority":1,"provider":"mock"}]}'::jsonb,'编排集成测试',
|
||||
'10000000-0000-0000-0000-000000000102','10000000-0000-0000-0000-000000000102','2026-08-23T11:00:00Z',
|
||||
'10000000-0000-0000-0000-000000000102','2026-08-23T11:01:00Z','10000000-0000-0000-0000-000000000102','2026-08-23T11:02:00Z'
|
||||
)`, []any{testRoutingConfigID, testRoutingConfigPublicID}},
|
||||
{`INSERT INTO billing_price_book (
|
||||
id,public_id,tenant_id,provider,currency,version_no,status,source_ref,change_summary,
|
||||
effective_from,created_by,published_by,published_at
|
||||
) VALUES (
|
||||
$1::uuid,'90000000-0000-0000-0000-000000000611','10000000-0000-0000-0000-000000000001',
|
||||
'mock','CNY',9001,'PUBLISHED','integration-test','provider usage integration test',
|
||||
'2026-01-01T00:00:00Z','10000000-0000-0000-0000-000000000102','10000000-0000-0000-0000-000000000102','2026-01-01T00:00:00Z'
|
||||
)`, []any{testPriceBookID}},
|
||||
{`INSERT INTO billing_price_rate (price_book_id,usage_category,metric_code,unit,unit_price_micros)
|
||||
VALUES ($1::uuid,'LLM','input_tokens','token',2)`, []any{testPriceBookID}},
|
||||
}
|
||||
for _, statement := range fixtureStatements {
|
||||
if _, err := pool.Exec(ctx, statement.query, statement.args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
configService, err := opsconfig.NewService(pool, audit.NewRepository())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configAdmin := platformidentity.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000101"}
|
||||
configApprover := platformidentity.Principal{Issuer: configAdmin.Issuer, Subject: "00000000-0000-0000-0000-000000000102"}
|
||||
bundle, created, err := configService.Create(ctx, configAdmin, testBundleCreateKey, "orchestration-agent-bundle-create", opsconfig.CreateInput{
|
||||
TenantID: "10000000-0000-0000-0000-000000000010", ConfigKey: testBundleConfigKey,
|
||||
DisplayName: "编排测试智能体包", ConfigKind: "AGENT_BUNDLE", ChangeSummary: "验证运营发布的自定义版本族进入通话运行时",
|
||||
Payload: map[string]any{
|
||||
"persona_version_id": testPersonaConfigPublicID, "voice_version_id": testVoiceConfigPublicID,
|
||||
"model_version_id": testModelConfigPublicID, "routing_version_id": testRoutingConfigPublicID,
|
||||
}, SecretRefs: map[string]string{},
|
||||
})
|
||||
if err != nil || !created {
|
||||
t.Fatalf("create Agent bundle=%+v created=%v err=%v", bundle, created, err)
|
||||
}
|
||||
bundle, err = configService.Submit(ctx, configAdmin, "10000000-0000-0000-0000-000000000010", bundle.ID, "orchestration-agent-bundle-submit", bundle.RowVersion)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
qualityMetrics := map[string]any{
|
||||
"report_schema": "agent-eval-report-v1", "suite_sha256": "b5cd3b798a01a48ddf0081f2fcc2a917f93681a3ee2fa55bbf4190466de3bef8",
|
||||
"agent_version": "orchestration-integration-v1", "model_config_ref": "runtime-config-version:" + testModelConfigPublicID,
|
||||
"config_version_id": bundle.ID, "environment": "mock", "executed_at": "2026-08-24T10:00:00Z",
|
||||
"cases_total": 10, "cases_passed": 10, "failed_check_count": 0, "conversation_success_rate": 1.0,
|
||||
"first_response_p95_ms": 1200, "interruption_recovery_rate": 1.0, "average_response_chars": 36.5,
|
||||
"salutation_accuracy": 1.0, "memory_accuracy": 1.0, "fabricated_facts": 0,
|
||||
"story_source_completeness": 1.0, "sensitive_information_handling_rate": 1.0,
|
||||
}
|
||||
if _, created, err = configService.CreateEvaluation(ctx, configAdmin, "10000000-0000-0000-0000-000000000010", bundle.ID,
|
||||
testBundleEvaluationKey, "orchestration-agent-bundle-evaluation", opsconfig.EvidenceTypeAgentQualityV1,
|
||||
"orchestration-agent-quality-v1", "PASSED", "artifact://orchestration-agent-quality.json", qualityMetrics); err != nil || !created {
|
||||
t.Fatalf("register Agent quality evidence created=%v err=%v", created, err)
|
||||
}
|
||||
bundle, err = configService.Publish(ctx, configApprover, "10000000-0000-0000-0000-000000000010", bundle.ID,
|
||||
"orchestration-agent-bundle-publish", opsconfig.PublishInput{Reason: "标准质量证据通过后灰度到测试长者", Rollout: opsconfig.RolloutInput{
|
||||
ScopeType: "SUBJECT", ScopeID: testSubjectPublicID, RolloutPercent: 100, MaxConcurrentCalls: 100,
|
||||
}}, bundle.RowVersion)
|
||||
if err != nil || bundle.Status != "PUBLISHED" {
|
||||
t.Fatalf("publish Agent bundle=%+v err=%v", bundle, err)
|
||||
}
|
||||
issuer, _ := rtc.NewLocalTokenIssuer([]byte("01234567890123456789012345678901"))
|
||||
service, err := NewService(
|
||||
idempotency.NewExecutor(pool), NewRepository(), audit.NewRepository(), outbox.NewStore(pool), issuer,
|
||||
testLeaseCache{},
|
||||
ServiceOptions{RTCAppID: "test-app", EngineMode: "s2s_hybrid", AllocationTTL: 30 * time.Second, TokenTTL: 5 * time.Minute, HeartbeatInterval: 15},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
embedder := &countingEmbedder{}
|
||||
if err = service.ConfigureMemoryEmbedding(pool, embedder); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service.now = func() time.Time { return time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) }
|
||||
identity := deviceauth.Identity{ID: testDeviceID, PublicID: testDevicePublicID, TenantID: "10000000-0000-0000-0000-000000000001", Status: "registered"}
|
||||
resume := true
|
||||
rtcCredentialRenewal := true
|
||||
input := CreateInput{DeviceTime: service.now(), Capabilities: Capabilities{RTC: true, AudioTracks: []string{"near"}, UploadResume: &resume, RTCCredentialRenewal: &rtcCredentialRenewal}, Network: Network{Type: "test"}, RequestedGoal: "open_conversation"}
|
||||
body := []byte(`{"device_time":"2026-08-23T12:00:00Z","capabilities":{"rtc":true,"audio_tracks":["near"],"upload_resume":true,"rtc_credential_renewal":true},"network":{"type":"test"},"requested_goal":"open_conversation"}`)
|
||||
first, replayed, err := service.Create(ctx, identity, "idempotency-test-0001", "request-1", body, input)
|
||||
if err != nil || replayed || first.RTC.Token == "" {
|
||||
t.Fatalf("first response=%#v replayed=%v err=%v", first, replayed, err)
|
||||
}
|
||||
var frozenContext, frozenBundleVersion string
|
||||
if err := pool.QueryRow(ctx, `SELECT cs.sanitized_summary,cs.prompt_bundle_version FROM context_snapshot cs JOIN conversation c ON c.context_snapshot_id=cs.id WHERE c.public_id=$1::uuid`, first.ConversationID).Scan(&frozenContext, &frozenBundleVersion); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if frozenBundleVersion != "cfg:"+bundle.ID || first.Session.EngineMode != "cascade" {
|
||||
t.Fatalf("frozen bundle=%q engine=%q", frozenBundleVersion, first.Session.EngineMode)
|
||||
}
|
||||
for _, expected := range []string{"服务测试长者", "年轻时从事铁路工作", "老照片", "聊聊第一次参加工作的经历", "放慢语速", "工作经历"} {
|
||||
if !strings.Contains(frozenContext, expected) {
|
||||
t.Fatalf("frozen context %q does not contain %q", frozenContext, expected)
|
||||
}
|
||||
}
|
||||
if strings.Contains(frozenContext, "不得进入语音上下文") || strings.Contains(frozenContext, "open_conversation") {
|
||||
t.Fatalf("frozen context exposed non-whitelisted or technical fields: %q", frozenContext)
|
||||
}
|
||||
second, replayed, err := service.Create(ctx, identity, "idempotency-test-0001", "request-2", body, input)
|
||||
if err != nil || !replayed || second.ConversationID != first.ConversationID || second.RTC.Token != first.RTC.Token {
|
||||
t.Fatalf("second response=%#v replayed=%v err=%v", second, replayed, err)
|
||||
}
|
||||
if _, _, err := service.Create(ctx, identity, "idempotency-test-0001", "request-3", append(body, ' '), input); !errors.Is(err, idempotency.ErrKeyConflict) {
|
||||
t.Fatalf("conflicting body error = %v", err)
|
||||
}
|
||||
if embedder.calls.Load() != 1 {
|
||||
t.Fatalf("embedding calls=%d want=1; sequential replay and body conflict must not repeat provider billing", embedder.calls.Load())
|
||||
}
|
||||
var conversations, histories, audits, events int
|
||||
var tokenPersisted bool
|
||||
if err := pool.QueryRow(ctx, "SELECT count(*) FROM conversation WHERE device_id=$1::uuid", testDeviceID).Scan(&conversations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM conversation_state_history h JOIN conversation c ON c.id=h.conversation_id WHERE c.device_id=$1::uuid`, testDeviceID).Scan(&histories); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, "SELECT count(*) FROM audit_event WHERE actor_public_id=$1::uuid AND action='conversation.create'", testDevicePublicID).Scan(&audits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event o JOIN conversation c ON c.id=o.aggregate_id WHERE c.device_id=$1::uuid AND o.event_type='conversation.room_issued'`, testDeviceID).Scan(&events); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM idempotency_record WHERE actor_key=$1 AND response_body::text LIKE '%dev1.%')`, "device:"+testDevicePublicID).Scan(&tokenPersisted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if conversations != 1 || histories != 2 || audits != 1 || events != 1 || tokenPersisted {
|
||||
t.Fatalf("conversation=%d history=%d audit=%d event=%d tokenPersisted=%v", conversations, histories, audits, events, tokenPersisted)
|
||||
}
|
||||
activateInput := ActivateInput{
|
||||
JoinedAt: service.now().Add(time.Second), RoomID: first.RTC.RoomID, RTCUserID: first.RTC.UserID,
|
||||
RTCJoinResult: "success", ConfigRevision: 1,
|
||||
}
|
||||
activateBody, _ := json.Marshal(activateInput)
|
||||
activated, replayed, err := service.Activate(ctx, identity, first.ConversationID, "activate-idempotency-0001", "activation-request", activateBody, activateInput)
|
||||
if err != nil || replayed || activated.Status != "STARTING_AGENT" {
|
||||
t.Fatalf("activation=%#v replayed=%v err=%v", activated, replayed, err)
|
||||
}
|
||||
replayedActivation, replayed, err := service.Activate(ctx, identity, first.ConversationID, "activate-idempotency-0001", "activation-request-replay", activateBody, activateInput)
|
||||
if err != nil || !replayed || replayedActivation != activated {
|
||||
t.Fatalf("replayed activation=%#v replayed=%v err=%v", replayedActivation, replayed, err)
|
||||
}
|
||||
var conversationInternalID, providerTaskRef string
|
||||
if err := pool.QueryRow(ctx, `SELECT id::text, provider_task_ref FROM conversation WHERE public_id=$1::uuid`, first.ConversationID).Scan(&conversationInternalID, &providerTaskRef); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
providerClient := &testVoiceProvider{}
|
||||
providerHealth, err := NewProviderHealth(pool, 3, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
startProcessor, err := NewStartProcessor(pool, providerClient, providerHealth, 75*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
startProcessor.now = func() time.Time { return service.now().Add(2 * time.Second) }
|
||||
if err := startProcessor.Handle(ctx, outbox.Event{AggregateID: conversationInternalID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if providerClient.startRequest.TaskRef != providerTaskRef || providerClient.startRequest.ConversationID != first.ConversationID || providerClient.startRequest.RoomID != first.RTC.RoomID || providerClient.startRequest.EngineMode != first.Session.EngineMode {
|
||||
t.Fatalf("provider start request=%#v", providerClient.startRequest)
|
||||
}
|
||||
if providerClient.startRequest.ContextSummary != frozenContext {
|
||||
t.Fatalf("provider context=%q want frozen context=%q", providerClient.startRequest.ContextSummary, frozenContext)
|
||||
}
|
||||
if providerClient.startRequest.PromptBundleVersion != frozenBundleVersion ||
|
||||
providerClient.startRequest.AgentBundleVersion != bundle.ID ||
|
||||
providerClient.startRequest.PersonaVersion != testPersonaConfigPublicID ||
|
||||
providerClient.startRequest.VoiceVersion != testVoiceConfigPublicID ||
|
||||
providerClient.startRequest.ModelVersion != testModelConfigPublicID ||
|
||||
providerClient.startRequest.RoutingVersion != testRoutingConfigPublicID ||
|
||||
providerClient.startRequest.SystemPrompt != "你是已审核发布的陪伴助手。" ||
|
||||
providerClient.startRequest.PersonaTone != "warm" ||
|
||||
providerClient.startRequest.VoiceResourceRef != "voice://orchestration/approved" ||
|
||||
providerClient.startRequest.ModelResourceRef != "model://orchestration/approved" {
|
||||
t.Fatalf("provider bundle request=%#v", providerClient.startRequest)
|
||||
}
|
||||
processor, _ := NewProviderEventProcessor(pool, 75*time.Second)
|
||||
processor.now = func() time.Time { return service.now().Add(3 * time.Second) }
|
||||
callback := voiceprovider.CallbackEvent{
|
||||
Provider: "mock", EventID: "provider-event-started-1", TaskRef: providerTaskRef,
|
||||
ConversationID: first.ConversationID, EventType: "voice_chat.started", Status: "ACTIVE",
|
||||
OccurredAt: service.now().Add(2500 * time.Millisecond),
|
||||
}
|
||||
rawCallback, _ := json.Marshal(callback)
|
||||
processed, err := processor.Process(ctx, callback, rawCallback)
|
||||
if err != nil || processed.Status != "ACTIVE" || processed.Duplicate || processed.Ignored {
|
||||
t.Fatalf("processed=%#v err=%v", processed, err)
|
||||
}
|
||||
duplicateCallback, err := processor.Process(ctx, callback, rawCallback)
|
||||
if err != nil || !duplicateCallback.Duplicate || duplicateCallback.Status != "ACTIVE" {
|
||||
t.Fatalf("duplicate=%#v err=%v", duplicateCallback, err)
|
||||
}
|
||||
conflictingCallback := callback
|
||||
conflictingCallback.Status = "FAILED"
|
||||
conflictingBody, _ := json.Marshal(conflictingCallback)
|
||||
if _, err := processor.Process(ctx, conflictingCallback, conflictingBody); !errors.Is(err, ErrProviderEventConflict) {
|
||||
t.Fatalf("conflicting callback error=%v", err)
|
||||
}
|
||||
callback.EventID = "provider-event-started-late"
|
||||
lateBody, _ := json.Marshal(callback)
|
||||
lateCallback, err := processor.Process(ctx, callback, lateBody)
|
||||
if err != nil || !lateCallback.Ignored || lateCallback.Status != "ACTIVE" {
|
||||
t.Fatalf("late=%#v err=%v", lateCallback, err)
|
||||
}
|
||||
var finalStatus, finalTaskStatus string
|
||||
var providerEvents int
|
||||
if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.public_id=$1::uuid`, first.ConversationID).Scan(&finalStatus, &finalTaskStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM provider_event e JOIN conversation c ON c.id=e.conversation_id WHERE c.public_id=$1::uuid`, first.ConversationID).Scan(&providerEvents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM conversation_state_history h JOIN conversation c ON c.id=h.conversation_id WHERE c.public_id=$1::uuid`, first.ConversationID).Scan(&histories); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if finalStatus != "ACTIVE" || finalTaskStatus != "ACTIVE" || providerEvents != 2 || histories != 4 {
|
||||
t.Fatalf("conversation=%s task=%s events=%d history=%d", finalStatus, finalTaskStatus, providerEvents, histories)
|
||||
}
|
||||
heartbeatOptions := HeartbeatServiceOptions{
|
||||
RTCAppID: "test-app", LeaseTTL: time.Minute, TokenTTL: 5 * time.Minute, CredentialRenewalWindow: 30 * time.Second,
|
||||
}
|
||||
heartbeatService, err := NewHeartbeatService(pool, NewRepository(), testLeaseCache{}, issuer, heartbeatOptions)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heartbeatService.now = func() time.Time { return time.Date(2026, 8, 23, 12, 0, 3, 500*1e6, time.UTC) }
|
||||
oldCredentialExpiry := heartbeatService.now().Add(20 * time.Second)
|
||||
if _, err := pool.Exec(ctx, `UPDATE rtc_task SET credential_issued_at=$2,credential_expires_at=$3 WHERE conversation_id=$1::uuid`, conversationInternalID, service.now(), oldCredentialExpiry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
monotonicMS := int64(12345)
|
||||
heartbeatInput := HeartbeatInput{SequenceNo: 1, DeviceTime: heartbeatService.now(), MonotonicMS: &monotonicMS, ConversationStatus: "ACTIVE", RTC: RTCHeartbeat{Joined: true, Uplink: true, Downlink: true}, Network: Network{Type: "wifi"}, FirmwareVersion: "1.0.0", ReportedConfigRevision: 1, RecordingQueue: RecordingQueue{}, RTCCredentialExpiresAt: &oldCredentialExpiry, CommandResults: []json.RawMessage{}}
|
||||
failingHeartbeatService, err := NewHeartbeatService(pool, NewRepository(), testLeaseCache{}, failingTokenIssuer{}, heartbeatOptions)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failingHeartbeatService.now = heartbeatService.now
|
||||
if _, err = pool.Exec(ctx, `UPDATE conversation SET rtc_credential_renewal_supported=false WHERE id=$1::uuid`, conversationInternalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacyHeartbeat, err := failingHeartbeatService.Heartbeat(ctx, identity, first.ConversationID, heartbeatInput)
|
||||
if err != nil || len(legacyHeartbeat.Actions) != 1 || legacyHeartbeat.Actions[0].Type != "continue" {
|
||||
t.Fatalf("legacy heartbeat=%#v err=%v", legacyHeartbeat, err)
|
||||
}
|
||||
var legacyExpiry time.Time
|
||||
if err = pool.QueryRow(ctx, `SELECT credential_expires_at FROM rtc_task WHERE conversation_id=$1::uuid`, conversationInternalID).Scan(&legacyExpiry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !legacyExpiry.Equal(oldCredentialExpiry) {
|
||||
t.Fatalf("legacy client rotated credential metadata: expiry=%s", legacyExpiry)
|
||||
}
|
||||
if _, err = pool.Exec(ctx, `UPDATE conversation SET rtc_credential_renewal_supported=true,last_heartbeat_seq=0 WHERE id=$1::uuid`, conversationInternalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = failingHeartbeatService.Heartbeat(ctx, identity, first.ConversationID, heartbeatInput); err == nil {
|
||||
t.Fatal("credential signing failure did not abort heartbeat")
|
||||
}
|
||||
var expiryAfterFailure time.Time
|
||||
var sequenceAfterFailure int64
|
||||
if err = pool.QueryRow(ctx, `SELECT r.credential_expires_at,c.last_heartbeat_seq FROM rtc_task r JOIN conversation c ON c.id=r.conversation_id WHERE c.id=$1::uuid`, conversationInternalID).Scan(&expiryAfterFailure, &sequenceAfterFailure); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !expiryAfterFailure.Equal(oldCredentialExpiry) || sequenceAfterFailure != 0 {
|
||||
t.Fatalf("failed renewal committed metadata expiry=%s sequence=%d", expiryAfterFailure, sequenceAfterFailure)
|
||||
}
|
||||
heartbeat, err := heartbeatService.Heartbeat(ctx, identity, first.ConversationID, heartbeatInput)
|
||||
if err != nil || heartbeat.LeaseExpiresAt == nil || len(heartbeat.Actions) != 2 || heartbeat.Actions[0].Type != "continue" || heartbeat.Actions[1].Type != "renew_rtc_credential" || heartbeat.Actions[1].RTC == nil || heartbeat.Actions[1].RTC.Token == "" || !heartbeat.Actions[1].RTC.ExpiresAt.Equal(heartbeatService.now().Add(5*time.Minute)) {
|
||||
t.Fatalf("heartbeat=%#v err=%v", heartbeat, err)
|
||||
}
|
||||
staleHeartbeat, err := heartbeatService.Heartbeat(ctx, identity, first.ConversationID, heartbeatInput)
|
||||
if err != nil || staleHeartbeat.LeaseExpiresAt == nil || !staleHeartbeat.LeaseExpiresAt.Equal(*heartbeat.LeaseExpiresAt) || len(staleHeartbeat.Actions) != 2 || staleHeartbeat.Actions[1].RTC == nil || staleHeartbeat.Actions[1].RTC.Token != heartbeat.Actions[1].RTC.Token {
|
||||
t.Fatalf("stale heartbeat=%#v err=%v", staleHeartbeat, err)
|
||||
}
|
||||
heartbeatInput.SequenceNo = 2
|
||||
heartbeatInput.RTCCredentialExpiresAt = &heartbeat.Actions[1].RTC.ExpiresAt
|
||||
heartbeatInput.ReportedConfigRevision = 2
|
||||
if _, err := heartbeatService.Heartbeat(ctx, identity, first.ConversationID, heartbeatInput); !errors.Is(err, ErrConfigVersionConflict) {
|
||||
t.Fatalf("future heartbeat configuration error=%v", err)
|
||||
}
|
||||
service.now = func() time.Time { return time.Date(2026, 8, 23, 12, 0, 4, 0, time.UTC) }
|
||||
stopInput := StopInput{
|
||||
Reason: "handset_on_hook", DeviceTime: service.now(), LastHeartbeatSequence: 2,
|
||||
RTCLeft: true, Recording: StopRecording{NearSegments: 1, FarSegments: 1},
|
||||
}
|
||||
stopBody := []byte(`{"reason":"handset_on_hook","device_time":"2026-08-23T12:00:04Z","last_heartbeat_sequence":2,"rtc_left":true,"recording":{"near_segments":1,"far_segments":1}}`)
|
||||
stopped, replayed, err := service.Stop(ctx, identity, first.ConversationID, "stop-idempotency-0001", "stop-request-1", stopBody, stopInput)
|
||||
if err != nil || replayed || stopped.Status != "STOPPING" {
|
||||
t.Fatalf("stop=%#v replayed=%v err=%v", stopped, replayed, err)
|
||||
}
|
||||
replayedStop, replayed, err := service.Stop(ctx, identity, first.ConversationID, "stop-idempotency-0001", "stop-request-2", stopBody, stopInput)
|
||||
if err != nil || !replayed || replayedStop.Status != "STOPPING" {
|
||||
t.Fatalf("replayed stop=%#v replayed=%v err=%v", replayedStop, replayed, err)
|
||||
}
|
||||
var stopEvents, stopAudits int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event o JOIN conversation c ON c.id=o.aggregate_id
|
||||
WHERE c.public_id=$1::uuid AND o.event_type='conversation.agent_stop_requested'`, first.ConversationID).Scan(&stopEvents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM audit_event a JOIN conversation c ON c.id=a.resource_id
|
||||
WHERE c.public_id=$1::uuid AND a.action='conversation.stop'`, first.ConversationID).Scan(&stopAudits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stopEvents != 1 || stopAudits != 1 {
|
||||
t.Fatalf("stop events=%d audits=%d", stopEvents, stopAudits)
|
||||
}
|
||||
stopProcessor, err := NewStopProcessor(pool, providerClient, providerHealth)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stopProcessor.now = func() time.Time { return service.now().Add(500 * time.Millisecond) }
|
||||
if err := stopProcessor.Handle(ctx, outbox.Event{AggregateID: conversationInternalID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if providerClient.stopRequest.TaskRef != providerTaskRef || providerClient.stopRequest.ConversationID != first.ConversationID || providerClient.stopRequest.Reason != stopInput.Reason {
|
||||
t.Fatalf("provider stop request=%#v", providerClient.stopRequest)
|
||||
}
|
||||
processor.now = func() time.Time { return time.Date(2026, 8, 23, 12, 0, 5, 0, time.UTC) }
|
||||
stopCallback := voiceprovider.CallbackEvent{
|
||||
Provider: "mock", EventID: "provider-event-stopped-1", TaskRef: providerTaskRef,
|
||||
ConversationID: first.ConversationID, EventType: "voice_chat.stopped", Status: "COMPLETED",
|
||||
OccurredAt: service.now().Add(time.Second),
|
||||
}
|
||||
rawStopCallback, _ := json.Marshal(stopCallback)
|
||||
completed, err := processor.Process(ctx, stopCallback, rawStopCallback)
|
||||
if err != nil || completed.Status != "COMPLETED" || completed.Ignored {
|
||||
t.Fatalf("completed=%#v err=%v", completed, err)
|
||||
}
|
||||
duplicateStop, err := processor.Process(ctx, stopCallback, rawStopCallback)
|
||||
if err != nil || !duplicateStop.Duplicate || duplicateStop.Status != "COMPLETED" {
|
||||
t.Fatalf("duplicate stop=%#v err=%v", duplicateStop, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id
|
||||
WHERE c.public_id=$1::uuid`, first.ConversationID).Scan(&finalStatus, &finalTaskStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM conversation_state_history h JOIN conversation c ON c.id=h.conversation_id
|
||||
WHERE c.public_id=$1::uuid`, first.ConversationID).Scan(&histories); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if finalStatus != "COMPLETED" || finalTaskStatus != "COMPLETED" || histories != 6 {
|
||||
t.Fatalf("final conversation=%s task=%s history=%d", finalStatus, finalTaskStatus, histories)
|
||||
}
|
||||
|
||||
// Exercise the synchronous ACTIVE/COMPLETED provider responses in addition
|
||||
// to the asynchronous callback path above. This is the other supported
|
||||
// provider contract and must preserve the same durable state transitions.
|
||||
secondConversation, replayed, err := service.Create(ctx, identity, "idempotency-test-0002", "request-4", body, input)
|
||||
if err != nil || replayed {
|
||||
t.Fatalf("second conversation=%#v replayed=%v err=%v", secondConversation, replayed, err)
|
||||
}
|
||||
secondActivateInput := activateInput
|
||||
secondActivateInput.RoomID = secondConversation.RTC.RoomID
|
||||
secondActivateInput.RTCUserID = secondConversation.RTC.UserID
|
||||
secondActivateBody, _ := json.Marshal(secondActivateInput)
|
||||
if _, replayed, err := service.Activate(ctx, identity, secondConversation.ConversationID, "activate-idempotency-0002", "activation-request-2", secondActivateBody, secondActivateInput); err != nil || replayed {
|
||||
t.Fatalf("second activation replayed=%v err=%v", replayed, err)
|
||||
}
|
||||
var secondInternalID string
|
||||
if err := pool.QueryRow(ctx, `SELECT id::text FROM conversation WHERE public_id=$1::uuid`, secondConversation.ConversationID).Scan(&secondInternalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
providerClient.startStatus = "ACTIVE"
|
||||
secondStartProcessor, err := NewStartProcessor(pool, providerClient, providerHealth, 75*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := secondStartProcessor.ConfigureStartProtection(allowProviderStart{}, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondStartProcessor.now = func() time.Time { return service.now().Add(2 * time.Minute) }
|
||||
if err := secondStartProcessor.Handle(ctx, outbox.Event{AggregateID: secondInternalID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondStopInput := stopInput
|
||||
secondStopInput.DeviceTime = service.now().Add(2*time.Minute + time.Second)
|
||||
secondStopBody, _ := json.Marshal(secondStopInput)
|
||||
if _, replayed, err := service.Stop(ctx, identity, secondConversation.ConversationID, "stop-idempotency-0002", "stop-request-3", secondStopBody, secondStopInput); err != nil || replayed {
|
||||
t.Fatalf("second stop replayed=%v err=%v", replayed, err)
|
||||
}
|
||||
providerClient.stopStatus = "COMPLETED"
|
||||
secondStopProcessor, err := NewStopProcessor(pool, providerClient, providerHealth)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondStopProcessor.now = func() time.Time { return service.now().Add(2*time.Minute + 2*time.Second) }
|
||||
if err := secondStopProcessor.Handle(ctx, outbox.Event{AggregateID: secondInternalID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.id=$1::uuid`, secondInternalID).Scan(&finalStatus, &finalTaskStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if finalStatus != "COMPLETED" || finalTaskStatus != "COMPLETED" {
|
||||
t.Fatalf("second final conversation=%s task=%s", finalStatus, finalTaskStatus)
|
||||
}
|
||||
|
||||
usageCallback := voiceprovider.CallbackEvent{
|
||||
Provider: "mock", EventID: "provider-event-usage-1", TaskRef: providerTaskRef,
|
||||
ConversationID: first.ConversationID, EventType: "voice_chat.usage", Status: "FINAL",
|
||||
OccurredAt: service.now().Add(3 * time.Minute), Usage: []voiceprovider.UsageMetric{{
|
||||
UsageCategory: "LLM", MetricCode: "input_tokens", Quantity: "12", Unit: "token",
|
||||
}},
|
||||
}
|
||||
usageBody, _ := json.Marshal(usageCallback)
|
||||
usageResult, err := processor.Process(ctx, usageCallback, usageBody)
|
||||
if err != nil || usageResult.UsageAccepted != 1 {
|
||||
t.Fatalf("provider usage result=%#v err=%v", usageResult, err)
|
||||
}
|
||||
|
||||
// A provider can end an active call before a device stop request arrives.
|
||||
// Verify that callback-only termination creates both durable transitions and
|
||||
// schedules content extraction.
|
||||
providerEndedConversation, replayed, err := service.Create(ctx, identity, "idempotency-test-0003", "request-5", body, input)
|
||||
if err != nil || replayed {
|
||||
t.Fatalf("provider-ended conversation=%#v replayed=%v err=%v", providerEndedConversation, replayed, err)
|
||||
}
|
||||
providerEndedActivate := activateInput
|
||||
providerEndedActivate.RoomID = providerEndedConversation.RTC.RoomID
|
||||
providerEndedActivate.RTCUserID = providerEndedConversation.RTC.UserID
|
||||
providerEndedActivateBody, _ := json.Marshal(providerEndedActivate)
|
||||
if _, replayed, err := service.Activate(ctx, identity, providerEndedConversation.ConversationID, "activate-idempotency-0003", "activation-request-3", providerEndedActivateBody, providerEndedActivate); err != nil || replayed {
|
||||
t.Fatalf("provider-ended activation replayed=%v err=%v", replayed, err)
|
||||
}
|
||||
var providerEndedInternalID, providerEndedTaskRef string
|
||||
if err := pool.QueryRow(ctx, `SELECT id::text,provider_task_ref FROM conversation WHERE public_id=$1::uuid`, providerEndedConversation.ConversationID).Scan(&providerEndedInternalID, &providerEndedTaskRef); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := secondStartProcessor.Handle(ctx, outbox.Event{AggregateID: providerEndedInternalID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
providerEndedCallback := voiceprovider.CallbackEvent{
|
||||
Provider: "mock", EventID: "provider-event-ended-active", TaskRef: providerEndedTaskRef,
|
||||
ConversationID: providerEndedConversation.ConversationID, EventType: "voice_chat.stopped", Status: "COMPLETED",
|
||||
OccurredAt: service.now().Add(4 * time.Minute),
|
||||
}
|
||||
providerEndedBody, _ := json.Marshal(providerEndedCallback)
|
||||
providerEndedResult, err := processor.Process(ctx, providerEndedCallback, providerEndedBody)
|
||||
if err != nil || providerEndedResult.Status != "COMPLETED" || providerEndedResult.Ignored {
|
||||
t.Fatalf("provider-ended result=%#v err=%v", providerEndedResult, err)
|
||||
}
|
||||
|
||||
// A runtime failure moves an active conversation to STOPPING and repeated
|
||||
// failure delivery remains safe while retaining the first failure cause.
|
||||
failedConversation, replayed, err := service.Create(ctx, identity, "idempotency-test-0004", "request-6", body, input)
|
||||
if err != nil || replayed {
|
||||
t.Fatalf("failed conversation=%#v replayed=%v err=%v", failedConversation, replayed, err)
|
||||
}
|
||||
failedActivate := activateInput
|
||||
failedActivate.RoomID = failedConversation.RTC.RoomID
|
||||
failedActivate.RTCUserID = failedConversation.RTC.UserID
|
||||
failedActivateBody, _ := json.Marshal(failedActivate)
|
||||
if _, replayed, err := service.Activate(ctx, identity, failedConversation.ConversationID, "activate-idempotency-0004", "activation-request-4", failedActivateBody, failedActivate); err != nil || replayed {
|
||||
t.Fatalf("failed activation replayed=%v err=%v", replayed, err)
|
||||
}
|
||||
var failedInternalID, failedTaskRef string
|
||||
if err := pool.QueryRow(ctx, `SELECT id::text,provider_task_ref FROM conversation WHERE public_id=$1::uuid`, failedConversation.ConversationID).Scan(&failedInternalID, &failedTaskRef); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := secondStartProcessor.Handle(ctx, outbox.Event{AggregateID: failedInternalID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failedCallback := voiceprovider.CallbackEvent{
|
||||
Provider: "mock", EventID: "provider-event-runtime-failed-1", TaskRef: failedTaskRef,
|
||||
ConversationID: failedConversation.ConversationID, EventType: "voice_chat.failed", Status: "FAILED",
|
||||
OccurredAt: service.now().Add(5 * time.Minute),
|
||||
}
|
||||
failedBody, _ := json.Marshal(failedCallback)
|
||||
failedResult, err := processor.Process(ctx, failedCallback, failedBody)
|
||||
if err != nil || failedResult.Status != "STOPPING" || failedResult.Ignored {
|
||||
t.Fatalf("runtime failure result=%#v err=%v", failedResult, err)
|
||||
}
|
||||
failedCallback.EventID = "provider-event-runtime-failed-2"
|
||||
repeatedFailedBody, _ := json.Marshal(failedCallback)
|
||||
repeatedFailedResult, err := processor.Process(ctx, failedCallback, repeatedFailedBody)
|
||||
if err != nil || repeatedFailedResult.Status != "STOPPING" || repeatedFailedResult.Ignored {
|
||||
t.Fatalf("repeated runtime failure result=%#v err=%v", repeatedFailedResult, err)
|
||||
}
|
||||
failedStopProcessor, err := NewStopProcessor(pool, providerClient, providerHealth)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
providerClient.stopErr = &voiceprovider.Error{Operation: "stop", Temporary: true, Cause: errors.New("temporary test failure")}
|
||||
if err := failedStopProcessor.Handle(ctx, outbox.Event{AggregateID: failedInternalID}); err == nil {
|
||||
t.Fatal("temporary provider stop failure was not retried")
|
||||
}
|
||||
providerClient.stopErr = nil
|
||||
providerClient.stopStatus = "COMPLETED"
|
||||
if err := failedStopProcessor.Handle(ctx, outbox.Event{AggregateID: failedInternalID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Reconciliation after an ambiguous start must query first. A still-starting
|
||||
// response is retried, a confirmed 404 permits the same stable task to be
|
||||
// started again, and a later ACTIVE query closes the ambiguity safely.
|
||||
reconciledConversation, replayed, err := service.Create(ctx, identity, "idempotency-test-0005", "request-7", body, input)
|
||||
if err != nil || replayed {
|
||||
t.Fatalf("reconciled conversation=%#v replayed=%v err=%v", reconciledConversation, replayed, err)
|
||||
}
|
||||
reconciledActivate := activateInput
|
||||
reconciledActivate.RoomID = reconciledConversation.RTC.RoomID
|
||||
reconciledActivate.RTCUserID = reconciledConversation.RTC.UserID
|
||||
reconciledActivateBody, _ := json.Marshal(reconciledActivate)
|
||||
if _, replayed, err := service.Activate(ctx, identity, reconciledConversation.ConversationID, "activate-idempotency-0005", "activation-request-5", reconciledActivateBody, reconciledActivate); err != nil || replayed {
|
||||
t.Fatalf("reconciled activation replayed=%v err=%v", replayed, err)
|
||||
}
|
||||
var reconciledInternalID string
|
||||
if err := pool.QueryRow(ctx, `SELECT id::text FROM conversation WHERE public_id=$1::uuid`, reconciledConversation.ConversationID).Scan(&reconciledInternalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='UNKNOWN' WHERE conversation_id=$1::uuid`, reconciledInternalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
providerClient.queryStatus = "STARTING"
|
||||
if err := secondStartProcessor.Handle(ctx, outbox.Event{AggregateID: reconciledInternalID}); err == nil {
|
||||
t.Fatal("still-starting reconciliation was not retried")
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='UNKNOWN' WHERE conversation_id=$1::uuid`, reconciledInternalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
providerClient.queryErr = &voiceprovider.Error{Operation: "query", StatusCode: 404}
|
||||
providerClient.startErr = &voiceprovider.Error{Operation: "start", Temporary: true, Cause: errors.New("ambiguous start")}
|
||||
if err := secondStartProcessor.Handle(ctx, outbox.Event{AggregateID: reconciledInternalID}); err == nil {
|
||||
t.Fatal("temporary provider start failure was not retried")
|
||||
}
|
||||
providerClient.queryErr = nil
|
||||
providerClient.startErr = nil
|
||||
providerClient.queryStatus = "ACTIVE"
|
||||
if err := secondStartProcessor.Handle(ctx, outbox.Event{AggregateID: reconciledInternalID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reconciledStopInput := stopInput
|
||||
reconciledStopInput.DeviceTime = service.now().Add(6*time.Minute + time.Second)
|
||||
reconciledStopBody, _ := json.Marshal(reconciledStopInput)
|
||||
if _, replayed, err := service.Stop(ctx, identity, reconciledConversation.ConversationID, "stop-idempotency-0005", "stop-request-5", reconciledStopBody, reconciledStopInput); err != nil || replayed {
|
||||
t.Fatalf("reconciled stop replayed=%v err=%v", replayed, err)
|
||||
}
|
||||
if err := failedStopProcessor.Handle(ctx, outbox.Event{AggregateID: reconciledInternalID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A permanent provider start rejection is not retryable. It must close the
|
||||
// authoritative state immediately so the device can leave the RTC room with
|
||||
// a stable, non-sensitive failure code instead of timing out in STARTING_AGENT.
|
||||
rejectedConversation, replayed, err := service.Create(ctx, identity, "idempotency-test-0006", "request-8", body, input)
|
||||
if err != nil || replayed {
|
||||
t.Fatalf("rejected conversation=%#v replayed=%v err=%v", rejectedConversation, replayed, err)
|
||||
}
|
||||
rejectedActivate := activateInput
|
||||
rejectedActivate.RoomID = rejectedConversation.RTC.RoomID
|
||||
rejectedActivate.RTCUserID = rejectedConversation.RTC.UserID
|
||||
rejectedActivateBody, _ := json.Marshal(rejectedActivate)
|
||||
if _, replayed, err = service.Activate(ctx, identity, rejectedConversation.ConversationID, "activate-idempotency-0006", "activation-request-6", rejectedActivateBody, rejectedActivate); err != nil || replayed {
|
||||
t.Fatalf("rejected activation replayed=%v err=%v", replayed, err)
|
||||
}
|
||||
var rejectedInternalID string
|
||||
if err = pool.QueryRow(ctx, `SELECT id::text FROM conversation WHERE public_id=$1::uuid`, rejectedConversation.ConversationID).Scan(&rejectedInternalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
providerClient.startErr = &voiceprovider.Error{Operation: "start", StatusCode: 400, Cause: errors.New("invalid provider request")}
|
||||
if err = secondStartProcessor.Handle(ctx, outbox.Event{AggregateID: rejectedInternalID}); err != nil {
|
||||
t.Fatalf("permanent provider rejection was not durably consumed: %v", err)
|
||||
}
|
||||
providerClient.startErr = nil
|
||||
rejectedState, err := service.GetDeviceState(ctx, identity, rejectedConversation.ConversationID)
|
||||
if err != nil || rejectedState.Status != "FAILED" || rejectedState.Action != "leave_room" || rejectedState.FailureCode != "provider_start_rejected" {
|
||||
t.Fatalf("rejected device state=%#v err=%v", rejectedState, err)
|
||||
}
|
||||
var rejectedTaskStatus string
|
||||
var rejectedHistory, rejectedAudits int
|
||||
if err = pool.QueryRow(ctx, `SELECT status FROM rtc_task WHERE conversation_id=$1::uuid`, rejectedInternalID).Scan(&rejectedTaskStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = pool.QueryRow(ctx, `SELECT count(*) FROM conversation_state_history WHERE conversation_id=$1::uuid AND from_status='STARTING_AGENT' AND to_status='FAILED'`, rejectedInternalID).Scan(&rejectedHistory); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = pool.QueryRow(ctx, `SELECT count(*) FROM audit_event WHERE resource_id=$1::uuid AND action='conversation.agent_start_failed'`, rejectedInternalID).Scan(&rejectedAudits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rejectedTaskStatus != "FAILED" || rejectedHistory != 1 || rejectedAudits != 1 {
|
||||
t.Fatalf("rejected task=%s history=%d audits=%d", rejectedTaskStatus, rejectedHistory, rejectedAudits)
|
||||
}
|
||||
|
||||
transcriptService, err := NewTranscriptBatchService(pool, idempotency.NewExecutor(pool), audit.NewRepository(), outbox.NewStore(pool))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
providerTurn := TranscriptTurnInput{
|
||||
TurnKey: "provider-final-1", SequenceNo: 1, Speaker: "subject",
|
||||
StartAt: time.Date(2026, 8, 23, 12, 0, 1, 0, time.UTC), EndAt: time.Date(2026, 8, 23, 12, 0, 2, 0, time.UTC),
|
||||
FinalText: "供应商已经返回的终稿", Interrupted: false,
|
||||
}
|
||||
lateTranscript := voiceprovider.CallbackEvent{
|
||||
Provider: "mock", EventID: "provider-event-transcript-late-1", TaskRef: providerTaskRef,
|
||||
ConversationID: first.ConversationID, EventType: "voice_chat.transcript.final", Status: "FINAL",
|
||||
OccurredAt: service.now().Add(2 * time.Second), Turns: []voiceprovider.FinalTurn{{
|
||||
TurnKey: providerTurn.TurnKey, SequenceNo: providerTurn.SequenceNo, Speaker: providerTurn.Speaker,
|
||||
StartAt: providerTurn.StartAt, EndAt: providerTurn.EndAt, FinalText: providerTurn.FinalText,
|
||||
Interrupted: providerTurn.Interrupted, ProviderRef: "provider-turn-1",
|
||||
}},
|
||||
}
|
||||
lateTranscriptBody, _ := json.Marshal(lateTranscript)
|
||||
lateTranscriptResult, err := processor.Process(ctx, lateTranscript, lateTranscriptBody)
|
||||
if err != nil || lateTranscriptResult.TurnsAccepted != 1 || lateTranscriptResult.Status != "COMPLETED" {
|
||||
t.Fatalf("late transcript=%#v err=%v", lateTranscriptResult, err)
|
||||
}
|
||||
deviceTurn := TranscriptTurnInput{
|
||||
TurnKey: "device-final-2", SequenceNo: 2, Speaker: "agent",
|
||||
StartAt: time.Date(2026, 8, 23, 12, 0, 2, 0, time.UTC), EndAt: time.Date(2026, 8, 23, 12, 0, 3, 0, time.UTC),
|
||||
FinalText: "终端补传的缺失终稿", Interrupted: true,
|
||||
}
|
||||
transcriptInput := TranscriptBatchInput{Turns: []TranscriptTurnInput{providerTurn, deviceTurn}}
|
||||
transcriptBody, _ := json.Marshal(transcriptInput)
|
||||
transcriptResponse, replayed, err := transcriptService.Submit(ctx, identity, first.ConversationID, "transcript-idempotency-0001", "transcript-request-1", transcriptBody, transcriptInput)
|
||||
if err != nil || replayed || transcriptResponse.AcceptedCount != 1 || transcriptResponse.DuplicateCount != 1 {
|
||||
t.Fatalf("transcript response=%#v replayed=%v err=%v", transcriptResponse, replayed, err)
|
||||
}
|
||||
replayedTranscript, replayed, err := transcriptService.Submit(ctx, identity, first.ConversationID, "transcript-idempotency-0001", "transcript-request-2", transcriptBody, transcriptInput)
|
||||
if err != nil || !replayed || replayedTranscript != transcriptResponse {
|
||||
t.Fatalf("replayed transcript=%#v replayed=%v err=%v", replayedTranscript, replayed, err)
|
||||
}
|
||||
duplicateTranscript, replayed, err := transcriptService.Submit(ctx, identity, first.ConversationID, "transcript-idempotency-0002", "transcript-request-3", transcriptBody, transcriptInput)
|
||||
if err != nil || replayed || duplicateTranscript.AcceptedCount != 0 || duplicateTranscript.DuplicateCount != 2 {
|
||||
t.Fatalf("duplicate transcript=%#v replayed=%v err=%v", duplicateTranscript, replayed, err)
|
||||
}
|
||||
conflictingInput := TranscriptBatchInput{Turns: []TranscriptTurnInput{deviceTurn}}
|
||||
conflictingInput.Turns[0].FinalText = "不一致的终稿"
|
||||
conflictingBody, _ = json.Marshal(conflictingInput)
|
||||
if _, _, err := transcriptService.Submit(ctx, identity, first.ConversationID, "transcript-idempotency-0003", "transcript-request-4", conflictingBody, conflictingInput); err == nil {
|
||||
t.Fatal("conflicting transcript was accepted")
|
||||
} else {
|
||||
var conflict *TranscriptConflictError
|
||||
if !errors.As(err, &conflict) || len(conflict.Conflicts) != 1 {
|
||||
t.Fatalf("conflicting transcript error=%v", err)
|
||||
}
|
||||
}
|
||||
wrongDevice := identity
|
||||
wrongDevice.ID = "90000000-0000-0000-0000-000000000599"
|
||||
if _, _, err := transcriptService.Submit(ctx, wrongDevice, first.ConversationID, "transcript-idempotency-0004", "transcript-request-5", transcriptBody, transcriptInput); !errors.Is(err, ErrConversationNotFound) {
|
||||
t.Fatalf("wrong device transcript error=%v", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE conversation SET retention_mode='none' WHERE public_id=$1::uuid`, first.ConversationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := transcriptService.Submit(ctx, identity, first.ConversationID, "transcript-idempotency-0005", "transcript-request-6", transcriptBody, transcriptInput); !errors.Is(err, ErrTranscriptStorageNotAllowed) {
|
||||
t.Fatalf("retention-disabled transcript error=%v", err)
|
||||
}
|
||||
var turnCount, deviceTurnCount, transcriptAudits, extractionEvents, providerExtractionEvents int
|
||||
var extractionGeneration int64
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*),count(*) FILTER (WHERE source_type='device') FROM conversation_turn
|
||||
WHERE conversation_id=(SELECT id FROM conversation WHERE public_id=$1::uuid)`, first.ConversationID).Scan(&turnCount, &deviceTurnCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM audit_event WHERE actor_public_id=$1::uuid AND action='conversation.transcript_batch.submit'`, testDevicePublicID).Scan(&transcriptAudits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*),max((payload->>'generation_version')::bigint) FROM outbox_event WHERE aggregate_id=(SELECT id FROM conversation WHERE public_id=$1::uuid)
|
||||
AND event_type='conversation.content_extraction_requested'
|
||||
AND deduplication_key LIKE 'content-extraction-v1:%:device:%'`, first.ConversationID).Scan(&extractionEvents, &extractionGeneration); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=(SELECT id FROM conversation WHERE public_id=$1::uuid)
|
||||
AND event_type='conversation.content_extraction_requested'
|
||||
AND deduplication_key LIKE 'content-extraction-v1:%:provider:%'`, first.ConversationID).Scan(&providerExtractionEvents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if turnCount != 2 || deviceTurnCount != 1 || transcriptAudits != 2 || extractionEvents != 1 || providerExtractionEvents != 1 || extractionGeneration != 2 {
|
||||
t.Fatalf("turns=%d device=%d audits=%d device_extraction=%d provider_extraction=%d generation=%d", turnCount, deviceTurnCount, transcriptAudits, extractionEvents, providerExtractionEvents, extractionGeneration)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupOrchestrationFixture(t *testing.T, ctx context.Context, pool *pgxpool.Pool) {
|
||||
t.Helper()
|
||||
statements := []struct {
|
||||
query string
|
||||
arg string
|
||||
}{
|
||||
{`DELETE FROM runtime_config_rollout WHERE config_key=$1`, testBundleConfigKey},
|
||||
{`DELETE FROM ops_config_evaluation WHERE config_version_id IN (SELECT id FROM runtime_config_version WHERE config_key=$1)`, testBundleConfigKey},
|
||||
{`DELETE FROM runtime_config_version WHERE config_key=$1`, testBundleConfigKey},
|
||||
{`DELETE FROM audit_event WHERE request_id LIKE $1`, "orchestration-agent-bundle-%"},
|
||||
{`DELETE FROM idempotency_record WHERE idempotency_key=$1`, testBundleCreateKey},
|
||||
{`DELETE FROM idempotency_record WHERE idempotency_key=$1`, testBundleEvaluationKey},
|
||||
{`DELETE FROM runtime_config_version WHERE id=$1::uuid`, testPersonaConfigID},
|
||||
{`DELETE FROM runtime_config_version WHERE id=$1::uuid`, testVoiceConfigID},
|
||||
{`DELETE FROM runtime_config_version WHERE id=$1::uuid`, testModelConfigID},
|
||||
{`DELETE FROM runtime_config_version WHERE id=$1::uuid`, testRoutingConfigID},
|
||||
{`DELETE FROM conversation_turn WHERE conversation_id IN (SELECT id FROM conversation WHERE device_id=$1::uuid)`, testDeviceID},
|
||||
{`DELETE FROM provider_event WHERE conversation_id IN (SELECT id FROM conversation WHERE device_id=$1::uuid)`, testDeviceID},
|
||||
{`DELETE FROM conversation_state_history WHERE conversation_id IN (SELECT id FROM conversation WHERE device_id=$1::uuid)`, testDeviceID},
|
||||
{`DELETE FROM rtc_task WHERE conversation_id IN (SELECT id FROM conversation WHERE device_id=$1::uuid)`, testDeviceID},
|
||||
{`DELETE FROM outbox_event WHERE aggregate_id IN (SELECT id FROM conversation WHERE device_id=$1::uuid)`, testDeviceID},
|
||||
{`DELETE FROM audit_event WHERE resource_id IN (SELECT id FROM conversation WHERE device_id=$1::uuid)`, testDeviceID},
|
||||
{`DELETE FROM audit_event WHERE actor_public_id=$1::uuid`, testDevicePublicID},
|
||||
{`DELETE FROM idempotency_record WHERE actor_key=$1`, "device:" + testDevicePublicID},
|
||||
{`DELETE FROM conversation WHERE device_id=$1::uuid`, testDeviceID},
|
||||
{`DELETE FROM context_snapshot WHERE subject_id=$1::uuid`, testSubjectID},
|
||||
{`DELETE FROM consent_snapshot WHERE subject_id=$1::uuid`, testSubjectID},
|
||||
{`DELETE FROM consent_record WHERE subject_id=$1::uuid`, testSubjectID},
|
||||
{`DELETE FROM device_binding WHERE device_id=$1::uuid`, testDeviceID},
|
||||
{`DELETE FROM device_presence_state WHERE device_id=$1::uuid`, testDeviceID},
|
||||
{`DELETE FROM device_twin WHERE device_id=$1::uuid`, testDeviceID},
|
||||
{`DELETE FROM device WHERE id=$1::uuid`, testDeviceID},
|
||||
{`DELETE FROM subject_profile WHERE id=$1::uuid`, testSubjectID},
|
||||
{`DELETE FROM billing_price_book WHERE id=$1::uuid`, testPriceBookID},
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := pool.Exec(ctx, statement.query, statement.arg); err != nil {
|
||||
t.Fatalf("cleanup fixture: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/deviceauth"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var ErrTranscriptStorageNotAllowed = errors.New("transcript storage is not allowed")
|
||||
|
||||
type TranscriptTurnInput struct {
|
||||
TurnKey string `json:"turn_key"`
|
||||
SequenceNo int64 `json:"sequence_no"`
|
||||
Speaker string `json:"speaker"`
|
||||
StartAt time.Time `json:"start_at"`
|
||||
EndAt time.Time `json:"end_at"`
|
||||
FinalText string `json:"final_text"`
|
||||
Interrupted bool `json:"interrupted"`
|
||||
}
|
||||
|
||||
type TranscriptBatchInput struct {
|
||||
Turns []TranscriptTurnInput `json:"turns"`
|
||||
}
|
||||
|
||||
type TranscriptConflict struct {
|
||||
TurnKey string `json:"turn_key"`
|
||||
SequenceNo int64 `json:"sequence_no"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type TranscriptConflictError struct {
|
||||
Conflicts []TranscriptConflict
|
||||
}
|
||||
|
||||
func (e *TranscriptConflictError) Error() string {
|
||||
return "device transcript conflicts with stored final turns"
|
||||
}
|
||||
|
||||
type TranscriptBatchResponse struct {
|
||||
ConversationID string `json:"conversation_id"`
|
||||
AcceptedCount int `json:"accepted_count"`
|
||||
DuplicateCount int `json:"duplicate_count"`
|
||||
}
|
||||
|
||||
type TranscriptBatchService struct {
|
||||
pool *pgxpool.Pool
|
||||
executor *idempotency.Executor
|
||||
audit *audit.Repository
|
||||
outbox *outbox.Store
|
||||
}
|
||||
|
||||
func NewTranscriptBatchService(pool *pgxpool.Pool, executor *idempotency.Executor, auditRepository *audit.Repository, outboxStore *outbox.Store) (*TranscriptBatchService, error) {
|
||||
if pool == nil || executor == nil || auditRepository == nil || outboxStore == nil {
|
||||
return nil, errors.New("transcript batch dependencies are required")
|
||||
}
|
||||
return &TranscriptBatchService{pool: pool, executor: executor, audit: auditRepository, outbox: outboxStore}, nil
|
||||
}
|
||||
|
||||
func (s *TranscriptBatchService) Submit(ctx context.Context, identity deviceauth.Identity, conversationID, idempotencyKey, requestID string, rawRequest []byte, input TranscriptBatchInput) (TranscriptBatchResponse, bool, error) {
|
||||
tenantID := identity.TenantID
|
||||
scope := "POST /device/v2/conversations/" + conversationID + "/turns:batch"
|
||||
result, err := s.executor.Execute(ctx, idempotency.Request{
|
||||
TenantID: &tenantID, ActorKey: "device:" + identity.PublicID, Scope: scope,
|
||||
Key: idempotencyKey, RequestHash: idempotency.Hash(rawRequest), ExpiresAfter: 24 * time.Hour,
|
||||
}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
response, internalID, submitErr := s.submit(operationContext, tx, identity, conversationID, input)
|
||||
if submitErr != nil {
|
||||
return idempotency.Result{}, submitErr
|
||||
}
|
||||
body, marshalErr := json.Marshal(response)
|
||||
if marshalErr != nil {
|
||||
return idempotency.Result{}, marshalErr
|
||||
}
|
||||
actorID := identity.PublicID
|
||||
if auditErr := s.audit.Record(operationContext, tx, audit.Event{
|
||||
TenantID: &tenantID, ActorType: "device", ActorPublicID: &actorID,
|
||||
Action: "conversation.transcript_batch.submit", ResourceType: "conversation", ResourceID: &internalID,
|
||||
RequestID: requestID, Result: "success", Detail: map[string]any{
|
||||
"accepted_count": response.AcceptedCount, "duplicate_count": response.DuplicateCount,
|
||||
},
|
||||
}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
if response.AcceptedCount > 0 {
|
||||
var generationVersion int64
|
||||
if versionErr := tx.QueryRow(operationContext, `SELECT COALESCE(max(sequence_no),1) FROM conversation_turn WHERE conversation_id=$1::uuid`, internalID).Scan(&generationVersion); versionErr != nil {
|
||||
return idempotency.Result{}, versionErr
|
||||
}
|
||||
deduplicationKey := "content-extraction-v1:" + internalID + ":device:" + idempotency.Hash(rawRequest)
|
||||
payload, marshalErr := json.Marshal(map[string]int64{"generation_version": generationVersion})
|
||||
if marshalErr != nil {
|
||||
return idempotency.Result{}, marshalErr
|
||||
}
|
||||
if _, enqueueErr := s.outbox.Enqueue(operationContext, tx, outbox.NewEvent{
|
||||
TenantID: &tenantID, AggregateType: "conversation", AggregateID: internalID,
|
||||
EventType: "conversation.content_extraction_requested", DeduplicationKey: &deduplicationKey,
|
||||
Payload: payload,
|
||||
}); enqueueErr != nil {
|
||||
return idempotency.Result{}, enqueueErr
|
||||
}
|
||||
}
|
||||
return idempotency.Result{StatusCode: http.StatusOK, Body: body, ResourceID: &internalID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return TranscriptBatchResponse{}, false, err
|
||||
}
|
||||
var response TranscriptBatchResponse
|
||||
if err := json.Unmarshal(result.Body, &response); err != nil {
|
||||
return TranscriptBatchResponse{}, result.Replayed, fmt.Errorf("decode transcript batch response: %w", err)
|
||||
}
|
||||
return response, result.Replayed, nil
|
||||
}
|
||||
|
||||
func (s *TranscriptBatchService) submit(ctx context.Context, tx pgx.Tx, identity deviceauth.Identity, conversationID string, input TranscriptBatchInput) (TranscriptBatchResponse, string, error) {
|
||||
var internalID, subjectID, retentionMode, status string
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT c.id::text,c.subject_id::text,c.retention_mode,c.status
|
||||
FROM conversation c
|
||||
WHERE c.public_id=$1::uuid AND c.tenant_id=$2::uuid AND c.device_id=$3::uuid
|
||||
FOR UPDATE`, conversationID, identity.TenantID, identity.ID).Scan(&internalID, &subjectID, &retentionMode, &status)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return TranscriptBatchResponse{}, "", ErrConversationNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return TranscriptBatchResponse{}, "", fmt.Errorf("read transcript conversation: %w", err)
|
||||
}
|
||||
if status != "ACTIVE" && status != "STOPPING" && status != "COMPLETED" && status != "FAILED" {
|
||||
return TranscriptBatchResponse{}, "", ErrConversationStateConflict
|
||||
}
|
||||
if retentionMode == "none" {
|
||||
return TranscriptBatchResponse{}, "", ErrTranscriptStorageNotAllowed
|
||||
}
|
||||
var hasConsentHistory, hasActiveConsent bool
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT
|
||||
EXISTS(SELECT 1 FROM consent_record WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND consent_type='transcript_storage'),
|
||||
EXISTS(SELECT 1 FROM consent_record WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND consent_type='transcript_storage' AND withdrawn_at IS NULL AND (expires_at IS NULL OR expires_at>now()))`,
|
||||
identity.TenantID, subjectID).Scan(&hasConsentHistory, &hasActiveConsent); err != nil {
|
||||
return TranscriptBatchResponse{}, "", err
|
||||
}
|
||||
if hasConsentHistory && !hasActiveConsent {
|
||||
return TranscriptBatchResponse{}, "", ErrTranscriptStorageNotAllowed
|
||||
}
|
||||
|
||||
response := TranscriptBatchResponse{ConversationID: conversationID}
|
||||
conflicts := make([]TranscriptConflict, 0)
|
||||
for _, turn := range input.Turns {
|
||||
rows, queryErr := tx.Query(ctx, `
|
||||
SELECT turn_key,sequence_no,speaker,start_at,end_at,final_text,interrupted
|
||||
FROM conversation_turn
|
||||
WHERE conversation_id=$1::uuid AND (turn_key=$2 OR sequence_no=$3)
|
||||
FOR UPDATE`, internalID, turn.TurnKey, turn.SequenceNo)
|
||||
if queryErr != nil {
|
||||
return TranscriptBatchResponse{}, "", queryErr
|
||||
}
|
||||
matches := 0
|
||||
exact := false
|
||||
for rows.Next() {
|
||||
matches++
|
||||
var stored TranscriptTurnInput
|
||||
if scanErr := rows.Scan(&stored.TurnKey, &stored.SequenceNo, &stored.Speaker, &stored.StartAt, &stored.EndAt, &stored.FinalText, &stored.Interrupted); scanErr != nil {
|
||||
rows.Close()
|
||||
return TranscriptBatchResponse{}, "", scanErr
|
||||
}
|
||||
if sameTranscriptTurn(stored, turn) {
|
||||
exact = true
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
if rows.Err() != nil {
|
||||
return TranscriptBatchResponse{}, "", rows.Err()
|
||||
}
|
||||
if matches > 0 {
|
||||
if matches == 1 && exact {
|
||||
response.DuplicateCount++
|
||||
continue
|
||||
}
|
||||
conflicts = append(conflicts, TranscriptConflict{TurnKey: turn.TurnKey, SequenceNo: turn.SequenceNo, Reason: "TURN_KEY_OR_SEQUENCE_ALREADY_USED"})
|
||||
continue
|
||||
}
|
||||
if _, insertErr := tx.Exec(ctx, `
|
||||
INSERT INTO conversation_turn(
|
||||
tenant_id,subject_id,conversation_id,turn_key,sequence_no,speaker,start_at,end_at,final_text,interrupted,source_type
|
||||
) VALUES($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,$8,$9,$10,'device')`,
|
||||
identity.TenantID, subjectID, internalID, turn.TurnKey, turn.SequenceNo, turn.Speaker,
|
||||
turn.StartAt, turn.EndAt, turn.FinalText, turn.Interrupted); insertErr != nil {
|
||||
return TranscriptBatchResponse{}, "", insertErr
|
||||
}
|
||||
response.AcceptedCount++
|
||||
}
|
||||
if len(conflicts) > 0 {
|
||||
return TranscriptBatchResponse{}, "", &TranscriptConflictError{Conflicts: conflicts}
|
||||
}
|
||||
return response, internalID, nil
|
||||
}
|
||||
|
||||
func sameTranscriptTurn(left, right TranscriptTurnInput) bool {
|
||||
return left.TurnKey == right.TurnKey && left.SequenceNo == right.SequenceNo && left.Speaker == right.Speaker &&
|
||||
left.StartAt.Equal(right.StartAt) && left.EndAt.Equal(right.EndAt) && left.FinalText == right.FinalText &&
|
||||
left.Interrupted == right.Interrupted
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/conversation"
|
||||
)
|
||||
|
||||
type CreateFailureError struct {
|
||||
ConversationID string
|
||||
FailureCode string
|
||||
}
|
||||
|
||||
func (e *CreateFailureError) Error() string {
|
||||
return fmt.Sprintf("conversation %s failed: %s", e.ConversationID, e.FailureCode)
|
||||
}
|
||||
|
||||
func (e *CreateFailureError) Unwrap() error { return ErrContextPreparationFailed }
|
||||
|
||||
type Capabilities struct {
|
||||
RTC bool `json:"rtc"`
|
||||
AudioTracks []string `json:"audio_tracks"`
|
||||
UploadResume *bool `json:"upload_resume"`
|
||||
RTCCredentialRenewal *bool `json:"rtc_credential_renewal,omitempty"`
|
||||
}
|
||||
|
||||
type Network struct {
|
||||
Type string `json:"type"`
|
||||
RSSI *int `json:"rssi,omitempty"`
|
||||
}
|
||||
|
||||
type CreateInput struct {
|
||||
DeviceTime time.Time `json:"device_time"`
|
||||
MonotonicMS *int64 `json:"monotonic_ms,omitempty"`
|
||||
Capabilities Capabilities `json:"capabilities"`
|
||||
Network Network `json:"network"`
|
||||
RequestedGoal string `json:"requested_goal"`
|
||||
}
|
||||
|
||||
type RTCGrant struct {
|
||||
AppID string `json:"app_id"`
|
||||
RoomID string `json:"room_id"`
|
||||
UserID string `json:"user_id"`
|
||||
AgentUserID string `json:"agent_user_id"`
|
||||
Token string `json:"token,omitempty"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
type Privacy struct {
|
||||
RetentionMode string `json:"retention_mode"`
|
||||
RecordNear bool `json:"record_near"`
|
||||
RecordFar bool `json:"record_far"`
|
||||
UploadAllowed bool `json:"upload_allowed"`
|
||||
}
|
||||
|
||||
type Session struct {
|
||||
EngineMode string `json:"engine_mode"`
|
||||
HeartbeatIntervalSecond int `json:"heartbeat_interval_sec"`
|
||||
AllocationExpiresAt time.Time `json:"allocation_expires_at"`
|
||||
}
|
||||
|
||||
type CreateResponse struct {
|
||||
ConversationID string `json:"conversation_id"`
|
||||
Status conversation.Status `json:"status"`
|
||||
FailureCode string `json:"failure_code,omitempty"`
|
||||
RTC RTCGrant `json:"rtc"`
|
||||
Privacy Privacy `json:"privacy"`
|
||||
Session Session `json:"session"`
|
||||
}
|
||||
|
||||
type ActivateInput struct {
|
||||
JoinedAt time.Time `json:"joined_at"`
|
||||
RoomID string `json:"room_id"`
|
||||
RTCUserID string `json:"rtc_user_id"`
|
||||
RTCJoinResult string `json:"rtc_join_result"`
|
||||
ConfigRevision int64 `json:"config_revision"`
|
||||
ConnectionReference string `json:"connection_reference,omitempty"`
|
||||
}
|
||||
|
||||
type ActivateResponse struct {
|
||||
ConversationID string `json:"conversation_id"`
|
||||
Status conversation.Status `json:"status"`
|
||||
AcceptedAt time.Time `json:"accepted_at"`
|
||||
}
|
||||
|
||||
type DeviceConversationState struct {
|
||||
ConversationID string `json:"conversation_id"`
|
||||
Status conversation.Status `json:"status"`
|
||||
Action string `json:"action"`
|
||||
FailureCode string `json:"failure_code,omitempty"`
|
||||
}
|
||||
|
||||
type RTCHeartbeat struct {
|
||||
Joined bool `json:"joined"`
|
||||
Uplink bool `json:"uplink"`
|
||||
Downlink bool `json:"downlink"`
|
||||
}
|
||||
|
||||
type RecordingQueue struct {
|
||||
PendingSegments int `json:"pending_segments"`
|
||||
FailedSegments int `json:"failed_segments"`
|
||||
LastFailureCode string `json:"last_failure_code,omitempty"`
|
||||
LastFailureAt *time.Time `json:"last_failure_at,omitempty"`
|
||||
}
|
||||
|
||||
type HeartbeatInput struct {
|
||||
SequenceNo int64 `json:"sequence_no"`
|
||||
DeviceTime time.Time `json:"device_time"`
|
||||
MonotonicMS *int64 `json:"monotonic_ms,omitempty"`
|
||||
ConversationStatus conversation.Status `json:"conversation_status"`
|
||||
RTC RTCHeartbeat `json:"rtc"`
|
||||
Network Network `json:"network"`
|
||||
FirmwareVersion string `json:"firmware_version"`
|
||||
ReportedConfigRevision int64 `json:"reported_config_revision"`
|
||||
RecordingQueue RecordingQueue `json:"recording_queue"`
|
||||
RTCCredentialExpiresAt *time.Time `json:"rtc_credential_expires_at,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
CommandResults []json.RawMessage `json:"command_results"`
|
||||
}
|
||||
|
||||
type HeartbeatAction struct {
|
||||
Type string `json:"type"`
|
||||
RTC *RTCGrant `json:"rtc,omitempty"`
|
||||
}
|
||||
|
||||
type HeartbeatResponse struct {
|
||||
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
|
||||
Actions []HeartbeatAction `json:"actions"`
|
||||
DesiredConfigRevision int64 `json:"desired_config_revision"`
|
||||
Commands []json.RawMessage `json:"commands"`
|
||||
}
|
||||
|
||||
type StopRecording struct {
|
||||
NearSegments int `json:"near_segments"`
|
||||
FarSegments int `json:"far_segments"`
|
||||
}
|
||||
|
||||
type StopInput struct {
|
||||
Reason string `json:"reason"`
|
||||
DeviceTime time.Time `json:"device_time"`
|
||||
LastHeartbeatSequence int64 `json:"last_heartbeat_sequence"`
|
||||
RTCLeft bool `json:"rtc_left"`
|
||||
Recording StopRecording `json:"recording"`
|
||||
}
|
||||
|
||||
type StopResponse struct {
|
||||
ConversationID string `json:"conversation_id"`
|
||||
Status conversation.Status `json:"status"`
|
||||
AcceptedAt time.Time `json:"accepted_at"`
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestConstructorAndBoundaryValidation(t *testing.T) {
|
||||
if _, err := NewStartProcessor(nil, nil, nil, 0); err == nil {
|
||||
t.Fatal("invalid start processor dependencies were accepted")
|
||||
}
|
||||
if err := (&StartProcessor{}).ConfigureStartProtection(nil, 0); err == nil {
|
||||
t.Fatal("invalid provider start protection was accepted")
|
||||
}
|
||||
if _, err := NewStopProcessor(nil, nil, nil); err == nil {
|
||||
t.Fatal("invalid stop processor dependencies were accepted")
|
||||
}
|
||||
if _, err := NewProviderEventProcessor(nil, 0); err == nil {
|
||||
t.Fatal("invalid provider event processor dependencies were accepted")
|
||||
}
|
||||
if _, err := NewProviderHealth(nil, 0, 0); err == nil {
|
||||
t.Fatal("invalid provider health dependencies were accepted")
|
||||
}
|
||||
if _, err := NewHeartbeatService(nil, nil, nil, nil, HeartbeatServiceOptions{}); err == nil {
|
||||
t.Fatal("invalid heartbeat dependencies were accepted")
|
||||
}
|
||||
if _, err := NewRecoveryScanner(nil, nil, nil, RecoveryOptions{}); err == nil {
|
||||
t.Fatal("invalid recovery scanner dependencies were accepted")
|
||||
}
|
||||
if _, err := NewTranscriptBatchService(nil, nil, nil, nil); err == nil {
|
||||
t.Fatal("invalid transcript dependencies were accepted")
|
||||
}
|
||||
if _, err := NewService(nil, nil, nil, nil, nil, nil, ServiceOptions{}); err == nil {
|
||||
t.Fatal("invalid orchestration dependencies were accepted")
|
||||
}
|
||||
|
||||
conflict := &TranscriptConflictError{}
|
||||
if !errors.Is(conflict, conflict) || conflict.Error() == "" {
|
||||
t.Fatal("transcript conflict error contract is invalid")
|
||||
}
|
||||
if got := truncateContextText(" 长者今日精神状态良好 ", 4); got != "长者今日…" {
|
||||
t.Fatalf("truncated context=%q", got)
|
||||
}
|
||||
if got := truncateContextText(" 简短 ", 10); got != "简短" {
|
||||
t.Fatalf("short context=%q", got)
|
||||
}
|
||||
if _, err := NewRedisLeaseCache(nil, "test"); err == nil {
|
||||
t.Fatal("nil redis client was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthorizedProfileContextRejectsMalformedWhitelistedFields(t *testing.T) {
|
||||
valid, err := authorizedProfileContext(
|
||||
[]byte(`{"background":"退休教师","interests":["戏曲","散步"],"internal_note":{"ignored":true}}`),
|
||||
[]byte(`{"conversation_goal":"轻松交流","pace":"slow","preferred_topics":["家乡"]}`),
|
||||
)
|
||||
if err != nil || valid.Background != "退休教师" || len(valid.Interests) != 2 || valid.Pace != "slow" {
|
||||
t.Fatalf("valid context=%+v err=%v", valid, err)
|
||||
}
|
||||
for _, malformed := range []struct {
|
||||
profile, preferences string
|
||||
}{
|
||||
{`{"background":42}`, `{}`},
|
||||
{`{"interests":["戏曲",42]}`, `{}`},
|
||||
{`{}`, `{"preferred_topics":"家乡"}`},
|
||||
} {
|
||||
if _, err := authorizedProfileContext([]byte(malformed.profile), []byte(malformed.preferences)); err == nil {
|
||||
t.Fatalf("malformed whitelisted context was accepted: profile=%s preferences=%s", malformed.profile, malformed.preferences)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user