chore: establish local platform baseline
This commit is contained in:
Codex Local
@@ -0,0 +1,754 @@
|
||||
package orchestration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/billing"
|
||||
"data-terminal-cloud/internal/conversation"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
"data-terminal-cloud/internal/ratelimit"
|
||||
"data-terminal-cloud/internal/voiceprovider"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrProviderTaskNotFound = errors.New("provider task was not found")
|
||||
ErrProviderEventConflict = errors.New("provider event key was reused with different content")
|
||||
ErrProviderRateLimited = errors.New("provider start rate limit reached")
|
||||
ErrProviderCapacity = errors.New("provider active task capacity reached")
|
||||
)
|
||||
|
||||
type providerStartRateLimiter interface {
|
||||
Allow(context.Context, string, string) (ratelimit.Result, error)
|
||||
}
|
||||
|
||||
type StartProcessor struct {
|
||||
pool *pgxpool.Pool
|
||||
provider voiceprovider.Client
|
||||
health *ProviderHealth
|
||||
initialLeaseTTL time.Duration
|
||||
startLimiter providerStartRateLimiter
|
||||
maxActiveTasks int
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// ConfigureStartProtection enables the provider-wide RPS bucket and durable
|
||||
// active-task capacity reservation used by every worker replica.
|
||||
func (p *StartProcessor) ConfigureStartProtection(limiter providerStartRateLimiter, maxActiveTasks int) error {
|
||||
if limiter == nil || maxActiveTasks <= 0 {
|
||||
return errors.New("provider start limiter and positive active task capacity are required")
|
||||
}
|
||||
p.startLimiter = limiter
|
||||
p.maxActiveTasks = maxActiveTasks
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewStartProcessor(pool *pgxpool.Pool, provider voiceprovider.Client, health *ProviderHealth, initialLeaseTTL time.Duration) (*StartProcessor, error) {
|
||||
if pool == nil || provider == nil || health == nil || initialLeaseTTL <= 0 {
|
||||
return nil, errors.New("database pool, voice provider client, provider health and positive lease TTL are required")
|
||||
}
|
||||
return &StartProcessor{pool: pool, provider: provider, health: health, initialLeaseTTL: initialLeaseTTL, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (p *StartProcessor) Handle(ctx context.Context, event outbox.Event) error {
|
||||
var request voiceprovider.StartRequest
|
||||
var conversationStatus conversation.Status
|
||||
var taskStatus, tenantID, providerName string
|
||||
err := p.pool.QueryRow(ctx, `
|
||||
SELECT c.public_id::text, c.status, r.provider_task_ref, r.room_id,
|
||||
r.device_rtc_user_id, r.agent_rtc_user_id, r.status,cs.sanitized_summary,
|
||||
c.engine_mode,c.tenant_id::text,r.provider,cs.prompt_bundle_version,
|
||||
COALESCE(bundle.public_id::text,''),COALESCE(bundle.payload->>'persona_version_id',''),
|
||||
COALESCE(bundle.payload->>'voice_version_id',''),COALESCE(bundle.payload->>'model_version_id',''),
|
||||
COALESCE(bundle.payload->>'routing_version_id',''),COALESCE(persona.payload->>'system_prompt',''),
|
||||
COALESCE(persona.payload->>'tone',''),COALESCE(voice.payload->>'voice_resource_ref',''),
|
||||
COALESCE(model.payload->>'model_id','')
|
||||
FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id
|
||||
JOIN context_snapshot cs ON cs.id=c.context_snapshot_id AND cs.tenant_id=c.tenant_id
|
||||
LEFT JOIN runtime_config_version bundle
|
||||
ON bundle.tenant_id=c.tenant_id AND bundle.config_kind='AGENT_BUNDLE'
|
||||
AND cs.prompt_bundle_version='cfg:' || bundle.public_id::text
|
||||
LEFT JOIN runtime_config_version persona
|
||||
ON persona.tenant_id=bundle.tenant_id AND persona.config_kind='PERSONA'
|
||||
AND persona.public_id::text=bundle.payload->>'persona_version_id'
|
||||
LEFT JOIN runtime_config_version voice
|
||||
ON voice.tenant_id=bundle.tenant_id AND voice.config_kind='VOICE'
|
||||
AND voice.public_id::text=bundle.payload->>'voice_version_id'
|
||||
LEFT JOIN runtime_config_version model
|
||||
ON model.tenant_id=bundle.tenant_id AND model.config_kind='MODEL'
|
||||
AND model.public_id::text=bundle.payload->>'model_version_id'
|
||||
WHERE c.id=$1::uuid`, event.AggregateID,
|
||||
).Scan(&request.ConversationID, &conversationStatus, &request.TaskRef, &request.RoomID,
|
||||
&request.DeviceUserID, &request.AgentUserID, &taskStatus, &request.ContextSummary, &request.EngineMode, &tenantID, &providerName,
|
||||
&request.PromptBundleVersion, &request.AgentBundleVersion, &request.PersonaVersion, &request.VoiceVersion,
|
||||
&request.ModelVersion, &request.RoutingVersion, &request.SystemPrompt, &request.PersonaTone,
|
||||
&request.VoiceResourceRef, &request.ModelResourceRef)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrProviderTaskNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read provider start task: %w", err)
|
||||
}
|
||||
if conversationStatus != conversation.StatusStartingAgent {
|
||||
return nil
|
||||
}
|
||||
if taskStatus != "STARTING" && taskStatus != "UNKNOWN" {
|
||||
return fmt.Errorf("provider task status %s cannot be started", taskStatus)
|
||||
}
|
||||
allowed, err := p.health.AllowStart(ctx, tenantID, providerName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read provider health: %w", err)
|
||||
}
|
||||
if !allowed {
|
||||
return outbox.Retryable(ErrProviderUnavailable)
|
||||
}
|
||||
var response voiceprovider.StartResponse
|
||||
if taskStatus == "UNKNOWN" {
|
||||
queryResult, queryErr := p.provider.Query(ctx, voiceprovider.QueryRequest{
|
||||
TaskRef: request.TaskRef, ConversationID: request.ConversationID,
|
||||
})
|
||||
switch {
|
||||
case queryErr == nil && queryResult.Status == "ACTIVE":
|
||||
response = voiceprovider.StartResponse{TaskRef: queryResult.TaskRef, Status: queryResult.Status}
|
||||
case queryErr == nil && queryResult.Status == "STARTING":
|
||||
_, _ = p.pool.Exec(ctx, `UPDATE rtc_task SET status='STARTING',last_checked_at=$2,updated_at=$2 WHERE conversation_id=$1::uuid`, event.AggregateID, p.now().UTC())
|
||||
return outbox.Retryable(errors.New("provider start is still pending confirmation"))
|
||||
case queryErr == nil:
|
||||
return fmt.Errorf("provider query returned unsupported start reconciliation status %s", queryResult.Status)
|
||||
case !voiceprovider.IsNotFound(queryErr):
|
||||
if voiceprovider.IsTemporary(queryErr) {
|
||||
return outbox.Retryable(fmt.Errorf("query unknown provider start before retry: %w", queryErr))
|
||||
}
|
||||
return fmt.Errorf("query unknown provider start before retry: %w", queryErr)
|
||||
}
|
||||
// A provider-confirmed 404 is the only UNKNOWN case in which Start may
|
||||
// safely be sent again. The same stable task reference is retained.
|
||||
}
|
||||
newReservation := false
|
||||
if response.Status == "" && p.maxActiveTasks > 0 {
|
||||
newReservation, err = p.reserveProviderCapacity(ctx, event.AggregateID, providerName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if response.Status == "" && p.startLimiter != nil {
|
||||
limitResult, limitErr := p.startLimiter.Allow(ctx, "provider_start", providerName)
|
||||
if limitErr != nil {
|
||||
return fmt.Errorf("apply provider start rate limit: %w", limitErr)
|
||||
}
|
||||
if !limitResult.Allowed {
|
||||
return outbox.Retryable(fmt.Errorf("%w: retry after %s", ErrProviderRateLimited, limitResult.RetryAfter))
|
||||
}
|
||||
}
|
||||
if response.Status == "" && !newReservation {
|
||||
_, err = p.pool.Exec(ctx, `
|
||||
UPDATE rtc_task SET start_attempts=start_attempts+1, last_checked_at=$2, updated_at=$2
|
||||
WHERE conversation_id=$1::uuid`, event.AggregateID, p.now().UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("record provider start attempt: %w", err)
|
||||
}
|
||||
}
|
||||
if response.Status == "" {
|
||||
response, err = p.provider.Start(ctx, request)
|
||||
if err != nil {
|
||||
if voiceprovider.IsTemporary(err) {
|
||||
_, _ = p.pool.Exec(ctx, `UPDATE rtc_task SET status='UNKNOWN', updated_at=$2 WHERE conversation_id=$1::uuid AND status IN ('STARTING','UNKNOWN')`, event.AggregateID, p.now().UTC())
|
||||
if healthErr := p.health.RecordFailure(ctx, tenantID, providerName, "PROVIDER_START_TEMPORARY"); healthErr != nil {
|
||||
return fmt.Errorf("record provider start failure: %v; provider error: %w", healthErr, err)
|
||||
}
|
||||
return outbox.Retryable(err)
|
||||
}
|
||||
return p.failRejectedStart(ctx, event.AggregateID, tenantID, "provider_start_rejected", "provider permanently rejected voice chat start")
|
||||
}
|
||||
}
|
||||
if err := p.health.RecordSuccess(ctx, tenantID, providerName); err != nil {
|
||||
return fmt.Errorf("record provider start success: %w", err)
|
||||
}
|
||||
if response.Status != "STARTING" && response.Status != "ACTIVE" {
|
||||
return p.failRejectedStart(ctx, event.AggregateID, tenantID, "provider_start_invalid_response", "provider returned an unsupported voice chat start status")
|
||||
}
|
||||
now := p.now().UTC()
|
||||
if response.Status == "STARTING" {
|
||||
_, err = p.pool.Exec(ctx, `
|
||||
UPDATE rtc_task SET status=CASE WHEN status='ACTIVE' THEN 'ACTIVE' ELSE 'STARTING' END,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, event.AggregateID, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record provider start response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
leaseExpiresAt := now.Add(p.initialLeaseTTL)
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='ACTIVE', lease_expires_at=$2,
|
||||
last_checked_at=$3, updated_at=$3 WHERE conversation_id=$1::uuid`, event.AggregateID, leaseExpiresAt, now); err != nil {
|
||||
return fmt.Errorf("record provider active response: %w", err)
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE conversation SET status='ACTIVE', started_at=COALESCE(started_at,$2),
|
||||
lease_expires_at=$3, version=version+1, updated_at=$2
|
||||
WHERE id=$1::uuid AND status='STARTING_AGENT'`, event.AggregateID, now, leaseExpiresAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile active conversation: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 1 {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source
|
||||
) VALUES ($1::uuid, 'STARTING_AGENT', 'ACTIVE', 'provider idempotent start response confirmed active task', 'worker')`, event.AggregateID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (p *StartProcessor) failRejectedStart(ctx context.Context, conversationID, tenantID, failureCode, reason string) error {
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
now := p.now().UTC()
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE conversation SET status='FAILED',ended_at=$2,lease_expires_at=NULL,
|
||||
failure_code=$3,version=version+1,updated_at=$2
|
||||
WHERE id=$1::uuid AND status='STARTING_AGENT'`, conversationID, now, failureCode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fail permanently rejected provider start: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE rtc_task SET status='FAILED',lease_expires_at=NULL,
|
||||
last_checked_at=$2,updated_at=$2 WHERE conversation_id=$1::uuid`, conversationID, now); err != nil {
|
||||
return fmt.Errorf("fail rejected provider RTC task: %w", err)
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id,from_status,to_status,reason,source
|
||||
) VALUES ($1::uuid,'STARTING_AGENT','FAILED',$2,'worker')`, conversationID, reason); err != nil {
|
||||
return fmt.Errorf("record rejected provider start state: %w", err)
|
||||
}
|
||||
detail, _ := json.Marshal(map[string]string{"failure_code": failureCode})
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO audit_event (
|
||||
tenant_id,actor_type,action,resource_type,resource_id,purpose,result,detail
|
||||
) VALUES ($1::uuid,'system','conversation.agent_start_failed','conversation',$2::uuid,
|
||||
'provider_start','failure',$3::jsonb)`, tenantID, conversationID, detail); err != nil {
|
||||
return fmt.Errorf("audit rejected provider start: %w", err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (p *StartProcessor) reserveProviderCapacity(ctx context.Context, conversationID, providerName string) (bool, error) {
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, "provider-capacity:"+providerName); err != nil {
|
||||
return false, fmt.Errorf("lock provider capacity: %w", err)
|
||||
}
|
||||
var attempts int
|
||||
if err = tx.QueryRow(ctx, `SELECT start_attempts FROM rtc_task WHERE conversation_id=$1::uuid FOR UPDATE`, conversationID).Scan(&attempts); err != nil {
|
||||
return false, fmt.Errorf("read provider capacity reservation: %w", err)
|
||||
}
|
||||
if attempts > 0 {
|
||||
return false, tx.Commit(ctx)
|
||||
}
|
||||
var active int
|
||||
if err = tx.QueryRow(ctx, `
|
||||
SELECT count(*) FROM rtc_task
|
||||
WHERE provider=$1 AND (
|
||||
status IN ('ACTIVE','STOPPING') OR
|
||||
(status IN ('STARTING','UNKNOWN') AND start_attempts>0)
|
||||
)`, providerName).Scan(&active); err != nil {
|
||||
return false, fmt.Errorf("count provider capacity: %w", err)
|
||||
}
|
||||
if active >= p.maxActiveTasks {
|
||||
return false, outbox.Retryable(fmt.Errorf("%w: provider=%s active=%d maximum=%d", ErrProviderCapacity, providerName, active, p.maxActiveTasks))
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE rtc_task SET start_attempts=1,last_checked_at=$2,updated_at=$2 WHERE conversation_id=$1::uuid`, conversationID, p.now().UTC()); err != nil {
|
||||
return false, fmt.Errorf("reserve provider capacity: %w", err)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type StopProcessor struct {
|
||||
pool *pgxpool.Pool
|
||||
provider voiceprovider.Client
|
||||
health *ProviderHealth
|
||||
outbox *outbox.Store
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewStopProcessor(pool *pgxpool.Pool, provider voiceprovider.Client, health *ProviderHealth) (*StopProcessor, error) {
|
||||
if pool == nil || provider == nil || health == nil {
|
||||
return nil, errors.New("database pool, voice provider client and provider health are required")
|
||||
}
|
||||
return &StopProcessor{pool: pool, provider: provider, health: health, outbox: outbox.NewStore(pool), now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (p *StopProcessor) Handle(ctx context.Context, event outbox.Event) error {
|
||||
var request voiceprovider.StopRequest
|
||||
var conversationStatus conversation.Status
|
||||
var taskStatus, tenantID, providerName string
|
||||
err := p.pool.QueryRow(ctx, `
|
||||
SELECT c.public_id::text, c.status, COALESCE(c.stop_reason,'provider_failure'), r.provider_task_ref, r.room_id, r.status, c.tenant_id::text, r.provider
|
||||
FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id
|
||||
WHERE c.id=$1::uuid`, event.AggregateID,
|
||||
).Scan(&request.ConversationID, &conversationStatus, &request.Reason, &request.TaskRef, &request.RoomID, &taskStatus, &tenantID, &providerName)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrProviderTaskNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read provider stop task: %w", err)
|
||||
}
|
||||
if conversationStatus != conversation.StatusStopping || taskStatus == "COMPLETED" {
|
||||
return nil
|
||||
}
|
||||
if taskStatus != "STOPPING" && taskStatus != "UNKNOWN" && taskStatus != "FAILED" {
|
||||
return fmt.Errorf("provider task status %s cannot be stopped", taskStatus)
|
||||
}
|
||||
now := p.now().UTC()
|
||||
_, err = p.pool.Exec(ctx, `
|
||||
UPDATE rtc_task SET stop_attempts=stop_attempts+1, last_checked_at=$2, updated_at=$2
|
||||
WHERE conversation_id=$1::uuid`, event.AggregateID, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record provider stop attempt: %w", err)
|
||||
}
|
||||
response, err := p.provider.Stop(ctx, request)
|
||||
if err != nil {
|
||||
_, _ = p.pool.Exec(ctx, `UPDATE rtc_task SET status='UNKNOWN', updated_at=$2
|
||||
WHERE conversation_id=$1::uuid AND status IN ('STOPPING','UNKNOWN','FAILED')`, event.AggregateID, p.now().UTC())
|
||||
if voiceprovider.IsTemporary(err) {
|
||||
if healthErr := p.health.RecordFailure(ctx, tenantID, providerName, "PROVIDER_STOP_TEMPORARY"); healthErr != nil {
|
||||
return fmt.Errorf("record provider stop failure: %v; provider error: %w", healthErr, err)
|
||||
}
|
||||
}
|
||||
if voiceprovider.IsTemporary(err) {
|
||||
return outbox.Retryable(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := p.health.RecordSuccess(ctx, tenantID, providerName); err != nil {
|
||||
return fmt.Errorf("record provider stop success: %w", err)
|
||||
}
|
||||
if response.Status != "STOPPING" && response.Status != "COMPLETED" {
|
||||
return fmt.Errorf("provider returned unsupported stop status %s", response.Status)
|
||||
}
|
||||
now = p.now().UTC()
|
||||
if response.Status == "STOPPING" {
|
||||
_, err = p.pool.Exec(ctx, `UPDATE rtc_task SET status=CASE WHEN status='COMPLETED' THEN 'COMPLETED' ELSE 'STOPPING' END,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, event.AggregateID, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record provider stop response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED', lease_expires_at=NULL,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, event.AggregateID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE conversation SET status='COMPLETED', ended_at=COALESCE(ended_at,$2),
|
||||
lease_expires_at=NULL, version=version+1, updated_at=$2 WHERE id=$1::uuid AND status='STOPPING'`, event.AggregateID, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 1 {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source
|
||||
) VALUES ($1::uuid, 'STOPPING', 'COMPLETED', 'provider idempotent stop response confirmed completed task', 'worker')`, event.AggregateID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
deduplicationKey := "content-extraction-v1:" + event.AggregateID
|
||||
if _, err := p.outbox.Enqueue(ctx, tx, outbox.NewEvent{
|
||||
TenantID: &tenantID, AggregateType: "conversation", AggregateID: event.AggregateID,
|
||||
EventType: "conversation.content_extraction_requested", DeduplicationKey: &deduplicationKey,
|
||||
Payload: json.RawMessage(`{"generation_version":1}`),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("enqueue conversation content extraction: %w", err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
type ProviderEventResult struct {
|
||||
ConversationID string `json:"conversation_id"`
|
||||
Status conversation.Status `json:"status"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
Ignored bool `json:"ignored"`
|
||||
TurnsAccepted int `json:"turns_accepted,omitempty"`
|
||||
UsageAccepted int `json:"usage_accepted,omitempty"`
|
||||
}
|
||||
|
||||
type ProviderEventProcessor struct {
|
||||
pool *pgxpool.Pool
|
||||
outbox *outbox.Store
|
||||
initialLeaseTTL time.Duration
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewProviderEventProcessor(pool *pgxpool.Pool, initialLeaseTTL time.Duration) (*ProviderEventProcessor, error) {
|
||||
if pool == nil || initialLeaseTTL <= 0 {
|
||||
return nil, errors.New("database pool and positive initial lease TTL are required")
|
||||
}
|
||||
return &ProviderEventProcessor{pool: pool, outbox: outbox.NewStore(pool), initialLeaseTTL: initialLeaseTTL, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (p *ProviderEventProcessor) Process(ctx context.Context, event voiceprovider.CallbackEvent, rawBody []byte) (ProviderEventResult, error) {
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var internalID, tenantID, subjectID, publicID, taskProvider, retentionMode string
|
||||
var currentStatus conversation.Status
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT c.id::text, c.tenant_id::text, c.subject_id::text, c.public_id::text, c.status, r.provider, c.retention_mode
|
||||
FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id
|
||||
WHERE r.provider_task_ref=$1
|
||||
FOR UPDATE OF c, r`, event.TaskRef,
|
||||
).Scan(&internalID, &tenantID, &subjectID, &publicID, ¤tStatus, &taskProvider, &retentionMode)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ProviderEventResult{}, ErrProviderTaskNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("read callback task: %w", err)
|
||||
}
|
||||
if event.Provider != taskProvider || event.ConversationID != publicID {
|
||||
return ProviderEventResult{}, ErrProviderTaskNotFound
|
||||
}
|
||||
digest := sha256.Sum256(rawBody)
|
||||
digestText := hex.EncodeToString(digest[:])
|
||||
eventID := uuid.NewString()
|
||||
sanitized, _ := json.Marshal(map[string]any{
|
||||
"event_type": event.EventType, "status": event.Status, "occurred_at": event.OccurredAt, "turn_count": len(event.Turns), "usage_count": len(event.Usage),
|
||||
})
|
||||
tag, err := tx.Exec(ctx, `
|
||||
INSERT INTO provider_event (
|
||||
id, tenant_id, conversation_id, provider, provider_event_key,
|
||||
provider_task_ref, event_type, occurred_at, payload_digest, sanitized_payload
|
||||
) VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10::jsonb)
|
||||
ON CONFLICT (provider, provider_event_key) DO NOTHING`,
|
||||
eventID, tenantID, internalID, event.Provider, event.EventID, event.TaskRef,
|
||||
event.EventType, event.OccurredAt, digestText, sanitized,
|
||||
)
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("store provider event: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
var storedDigest string
|
||||
if err := tx.QueryRow(ctx, `SELECT payload_digest FROM provider_event WHERE provider=$1 AND provider_event_key=$2`, event.Provider, event.EventID).Scan(&storedDigest); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if storedDigest != digestText {
|
||||
return ProviderEventResult{}, ErrProviderEventConflict
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return ProviderEventResult{ConversationID: publicID, Status: currentStatus, Duplicate: true}, nil
|
||||
}
|
||||
|
||||
result := ProviderEventResult{ConversationID: publicID, Status: currentStatus}
|
||||
if event.EventType == "voice_chat.usage" && event.Status == "FINAL" {
|
||||
metrics := make([]billing.UsageMetricInput, 0, len(event.Usage))
|
||||
for _, metric := range event.Usage {
|
||||
metrics = append(metrics, billing.UsageMetricInput{UsageCategory: metric.UsageCategory, MetricCode: metric.MetricCode, Quantity: metric.Quantity, Unit: metric.Unit})
|
||||
}
|
||||
recorded, usageErr := billing.RecordProviderUsageTx(ctx, tx, tenantID, internalID, eventID, event.Provider, event.EventID, event.OccurredAt, metrics)
|
||||
if errors.Is(usageErr, billing.ErrUsageConflict) {
|
||||
return ProviderEventResult{}, ErrProviderEventConflict
|
||||
}
|
||||
if usageErr != nil {
|
||||
return ProviderEventResult{}, usageErr
|
||||
}
|
||||
if _, usageErr = tx.Exec(ctx, `UPDATE provider_event SET processing_status='processed',processed_at=$2 WHERE id=$1::uuid`, eventID, p.now().UTC()); usageErr != nil {
|
||||
return ProviderEventResult{}, usageErr
|
||||
}
|
||||
if usageErr = tx.Commit(ctx); usageErr != nil {
|
||||
return ProviderEventResult{}, usageErr
|
||||
}
|
||||
result.UsageAccepted = len(recorded)
|
||||
return result, nil
|
||||
}
|
||||
if event.EventType == "voice_chat.transcript.final" && event.Status == "FINAL" {
|
||||
return p.processTranscriptEvent(ctx, tx, eventID, internalID, tenantID, subjectID, retentionMode, event, result)
|
||||
}
|
||||
if event.EventType == "voice_chat.stopped" && event.Status == "COMPLETED" {
|
||||
return p.completeStoppedEvent(ctx, tx, eventID, internalID, tenantID, currentStatus, result)
|
||||
}
|
||||
if event.EventType == "voice_chat.failed" && event.Status == "FAILED" {
|
||||
return p.processFailedEvent(ctx, tx, eventID, internalID, currentStatus, result)
|
||||
}
|
||||
if event.EventType != "voice_chat.started" || event.Status != "ACTIVE" || currentStatus != conversation.StatusStartingAgent {
|
||||
result.Ignored = true
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status='ignored', processed_at=$2 WHERE id=$1::uuid`, eventID, p.now().UTC()); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
if err := conversation.ValidateTransition(currentStatus, conversation.StatusActive); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
now := p.now().UTC()
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE conversation SET status='ACTIVE', started_at=COALESCE(started_at,$2),
|
||||
lease_expires_at=$3, version=version+1, updated_at=$2
|
||||
WHERE id=$1::uuid AND status='STARTING_AGENT'`, internalID, now, now.Add(p.initialLeaseTTL))
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("activate conversation from provider event: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE rtc_task SET status='ACTIVE', lease_expires_at=$2, last_checked_at=$3, updated_at=$3
|
||||
WHERE conversation_id=$1::uuid`, internalID, now.Add(p.initialLeaseTTL), now)
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("activate RTC task: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, provider_event_id
|
||||
) VALUES ($1::uuid, 'STARTING_AGENT', 'ACTIVE', 'provider confirmed voice chat start', 'provider', $2::uuid)`, internalID, eventID)
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("record provider active state: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE provider_event SET processing_status='processed', processed_at=$2 WHERE id=$1::uuid`, eventID, now)
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
result.Status = conversation.StatusActive
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *ProviderEventProcessor) processTranscriptEvent(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
eventID, conversationID, tenantID, subjectID, retentionMode string,
|
||||
event voiceprovider.CallbackEvent,
|
||||
result ProviderEventResult,
|
||||
) (ProviderEventResult, error) {
|
||||
now := p.now().UTC()
|
||||
if retentionMode == "none" {
|
||||
result.Ignored = true
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status='ignored', processed_at=$2 WHERE id=$1::uuid`, eventID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
for _, turn := range event.Turns {
|
||||
providerRef := turn.ProviderRef
|
||||
if providerRef == "" {
|
||||
providerRef = turn.TurnKey
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
INSERT INTO conversation_turn (
|
||||
tenant_id,subject_id,conversation_id,turn_key,sequence_no,speaker,start_at,end_at,
|
||||
final_text,interrupted,provider_ref,source_type
|
||||
) VALUES ($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,$8,$9,$10,$11,'provider')
|
||||
ON CONFLICT (conversation_id,turn_key) DO NOTHING`, tenantID, subjectID, conversationID,
|
||||
turn.TurnKey, turn.SequenceNo, turn.Speaker, turn.StartAt, turn.EndAt, turn.FinalText, turn.Interrupted, providerRef)
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("store final transcript turn: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
var sequenceNo int64
|
||||
var speaker, finalText string
|
||||
var startAt, endAt time.Time
|
||||
var interrupted bool
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT sequence_no,speaker,start_at,end_at,final_text,interrupted
|
||||
FROM conversation_turn WHERE conversation_id=$1::uuid AND turn_key=$2`, conversationID, turn.TurnKey).Scan(
|
||||
&sequenceNo, &speaker, &startAt, &endAt, &finalText, &interrupted); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if sequenceNo != turn.SequenceNo || speaker != turn.Speaker || !startAt.Equal(turn.StartAt) ||
|
||||
!endAt.Equal(turn.EndAt) || finalText != turn.FinalText || interrupted != turn.Interrupted {
|
||||
return ProviderEventResult{}, ErrProviderEventConflict
|
||||
}
|
||||
continue
|
||||
}
|
||||
result.TurnsAccepted++
|
||||
}
|
||||
// A provider may deliver its final transcript after the stopped callback.
|
||||
// In that ordering the original extraction job may already have run before
|
||||
// any turns existed, so accepted late turns must create a new idempotent
|
||||
// generation request rather than leaving the conversation without content.
|
||||
if result.TurnsAccepted > 0 && conversation.IsTerminal(result.Status) {
|
||||
var generationVersion int64
|
||||
if err := tx.QueryRow(ctx, `SELECT COALESCE(max(sequence_no),1) FROM conversation_turn WHERE conversation_id=$1::uuid`, conversationID).Scan(&generationVersion); err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("read late transcript generation: %w", err)
|
||||
}
|
||||
digest := sha256.Sum256([]byte(event.Provider + ":" + event.EventID))
|
||||
deduplicationKey := "content-extraction-v1:" + conversationID + ":provider:" + hex.EncodeToString(digest[:16])
|
||||
payload, err := json.Marshal(map[string]int64{"generation_version": generationVersion})
|
||||
if err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := p.outbox.Enqueue(ctx, tx, outbox.NewEvent{
|
||||
TenantID: &tenantID, AggregateType: "conversation", AggregateID: conversationID,
|
||||
EventType: "conversation.content_extraction_requested", DeduplicationKey: &deduplicationKey,
|
||||
Payload: payload,
|
||||
}); err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("enqueue late provider transcript extraction: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status='processed', processed_at=$2 WHERE id=$1::uuid`, eventID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *ProviderEventProcessor) completeStoppedEvent(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
eventID, internalID, tenantID string,
|
||||
currentStatus conversation.Status,
|
||||
result ProviderEventResult,
|
||||
) (ProviderEventResult, error) {
|
||||
if conversation.IsTerminal(currentStatus) || currentStatus == conversation.StatusRoomIssued || currentStatus == conversation.StatusPreparing {
|
||||
result.Ignored = true
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status='ignored', processed_at=$2 WHERE id=$1::uuid`, eventID, p.now().UTC()); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
now := p.now().UTC()
|
||||
if currentStatus == conversation.StatusStartingAgent || currentStatus == conversation.StatusActive {
|
||||
if err := conversation.ValidateTransition(currentStatus, conversation.StatusStopping); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET status='STOPPING', lease_expires_at=NULL,
|
||||
version=version+1, updated_at=$2 WHERE id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, provider_event_id
|
||||
) VALUES ($1::uuid, $2, 'STOPPING', 'provider ended task before platform stop completed', 'provider', $3::uuid)`,
|
||||
internalID, currentStatus, eventID); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET status='COMPLETED', ended_at=COALESCE(ended_at,$2),
|
||||
lease_expires_at=NULL, stop_reason=COALESCE(stop_reason,'provider_ended'),
|
||||
version=version+1, updated_at=$2 WHERE id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED', lease_expires_at=NULL,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, provider_event_id
|
||||
) VALUES ($1::uuid, 'STOPPING', 'COMPLETED', 'provider confirmed voice chat stop', 'provider', $2::uuid)`,
|
||||
internalID, eventID); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status='processed', processed_at=$2 WHERE id=$1::uuid`, eventID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
deduplicationKey := "content-extraction-v1:" + internalID
|
||||
if _, err := p.outbox.Enqueue(ctx, tx, outbox.NewEvent{
|
||||
TenantID: &tenantID, AggregateType: "conversation", AggregateID: internalID,
|
||||
EventType: "conversation.content_extraction_requested", DeduplicationKey: &deduplicationKey,
|
||||
Payload: json.RawMessage(`{"generation_version":1}`),
|
||||
}); err != nil {
|
||||
return ProviderEventResult{}, fmt.Errorf("enqueue conversation content extraction: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
result.Status = conversation.StatusCompleted
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *ProviderEventProcessor) processFailedEvent(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
eventID, internalID string,
|
||||
currentStatus conversation.Status,
|
||||
result ProviderEventResult,
|
||||
) (ProviderEventResult, error) {
|
||||
now := p.now().UTC()
|
||||
switch currentStatus {
|
||||
case conversation.StatusStartingAgent:
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET status='FAILED', ended_at=$2,
|
||||
lease_expires_at=NULL, failure_code='provider_start_failed', version=version+1, updated_at=$2
|
||||
WHERE id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, provider_event_id
|
||||
) VALUES ($1::uuid, 'STARTING_AGENT', 'FAILED', 'provider reported voice chat failure', 'provider', $2::uuid)`, internalID, eventID); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
result.Status = conversation.StatusFailed
|
||||
case conversation.StatusActive:
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET status='STOPPING', lease_expires_at=NULL,
|
||||
failure_code='provider_runtime_failed', version=version+1, updated_at=$2 WHERE id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
|
||||
conversation_id, from_status, to_status, reason, source, provider_event_id
|
||||
) VALUES ($1::uuid, 'ACTIVE', 'STOPPING', 'provider reported runtime failure', 'provider', $2::uuid)`, internalID, eventID); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
result.Status = conversation.StatusStopping
|
||||
case conversation.StatusStopping:
|
||||
if _, err := tx.Exec(ctx, `UPDATE conversation SET failure_code=COALESCE(failure_code,'provider_stop_failed'), updated_at=$2 WHERE id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
default:
|
||||
result.Ignored = true
|
||||
}
|
||||
if !result.Ignored {
|
||||
if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='FAILED', lease_expires_at=NULL,
|
||||
last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, internalID, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
}
|
||||
processingStatus := "processed"
|
||||
if result.Ignored {
|
||||
processingStatus = "ignored"
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE provider_event SET processing_status=$2, processed_at=$3 WHERE id=$1::uuid`, eventID, processingStatus, now); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProviderEventResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
Reference in New Issue
Block a user