chore: checkpoint 第一版

2026-09-06 17:04:11 +08:00
parent a8cdec98d1
commit 0c2151ace9
659 changed files with 185855 additions and 554 deletions
@@ -39,6 +39,7 @@ type recoveryCandidate struct {
 	ProviderTaskRef string
 	StopReason      string
 	Trigger         string
+	RuntimeType     string
 }

 func NewRecoveryScanner(pool *pgxpool.Pool, outboxStore *outbox.Store, logger *slog.Logger, options RecoveryOptions) (*RecoveryScanner, error) {
@@ -81,8 +82,9 @@ func (s *RecoveryScanner) ScanOnce(ctx context.Context) (int, error) {
 			CASE WHEN c.status='ACTIVE' AND cub.in_progress_action='STOP_IMMEDIATELY'
 			          AND c.started_at IS NOT NULL
 			          AND c.started_at+make_interval(secs=>cub.max_call_duration_sec)<=$1
-			     THEN 'quota_duration_limit' ELSE '' END
-		FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id
+			     THEN 'quota_duration_limit' ELSE '' END,c.runtime_type
+		FROM conversation c LEFT JOIN rtc_task r ON r.conversation_id=c.id
+		LEFT JOIN hardware_agent_session hs ON hs.conversation_id=c.id
 		LEFT JOIN conversation_usage_budget cub ON cub.conversation_id=c.id AND cub.tenant_id=c.tenant_id
 		WHERE (c.status='ROOM_ISSUED' AND c.allocation_expires_at <= $1)
 		   OR (c.status='ACTIVE' AND c.lease_expires_at IS NOT NULL AND c.lease_expires_at <= $1)
@@ -90,7 +92,7 @@ func (s *RecoveryScanner) ScanOnce(ctx context.Context) (int, error) {
 		   OR (c.status='STARTING_AGENT' AND c.updated_at <= $2)
 		   OR (c.status='STOPPING' AND c.updated_at <= $3)
 		ORDER BY c.updated_at, c.id
-		LIMIT $4 FOR UPDATE OF c, r SKIP LOCKED`,
+		LIMIT $4 FOR UPDATE OF c SKIP LOCKED`,
 		now, now.Add(-s.options.StartingAfter), now.Add(-s.options.StoppingAfter), s.options.BatchSize)
 	if err != nil {
 		return 0, fmt.Errorf("select conversations for recovery: %w", err)
@@ -98,7 +100,7 @@ func (s *RecoveryScanner) ScanOnce(ctx context.Context) (int, error) {
 	candidates := make([]recoveryCandidate, 0, s.options.BatchSize)
 	for rows.Next() {
 		var candidate recoveryCandidate
-		if err := rows.Scan(&candidate.InternalID, &candidate.PublicID, &candidate.TenantID, &candidate.Status, &candidate.ProviderTaskRef, &candidate.StopReason, &candidate.Trigger); err != nil {
+		if err := rows.Scan(&candidate.InternalID, &candidate.PublicID, &candidate.TenantID, &candidate.Status, &candidate.ProviderTaskRef, &candidate.StopReason, &candidate.Trigger, &candidate.RuntimeType); err != nil {
 			rows.Close()
 			return 0, err
 		}
@@ -130,9 +132,15 @@ func (s *RecoveryScanner) recoverCandidate(ctx context.Context, tx pgx.Tx, candi
 			WHERE id=$1::uuid AND status='ROOM_ISSUED'`, candidate.InternalID, now); err != nil {
 			return err
 		}
-		if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED', lease_expires_at=NULL,
-			last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, candidate.InternalID, now); err != nil {
-			return err
+		if candidate.RuntimeType == RuntimeVolcHardwareAgent {
+			if _, err := tx.Exec(ctx, `UPDATE hardware_agent_session SET status='EXPIRED',updated_at=$2 WHERE conversation_id=$1::uuid`, candidate.InternalID, now); err != nil {
+				return err
+			}
+		} else {
+			if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED', lease_expires_at=NULL,
+				last_checked_at=$2, updated_at=$2 WHERE conversation_id=$1::uuid`, candidate.InternalID, now); err != nil {
+				return err
+			}
 		}
 		if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
 			conversation_id, from_status, to_status, reason, source
@@ -153,17 +161,25 @@ func (s *RecoveryScanner) recoverCandidate(ctx context.Context, tx pgx.Tx, candi
 			WHERE id=$1::uuid AND status='ACTIVE'`, candidate.InternalID, now, reason); err != nil {
 			return err
 		}
-		if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='STOPPING', lease_expires_at=NULL,
-			updated_at=$2 WHERE conversation_id=$1::uuid`, candidate.InternalID, now); err != nil {
-			return err
+		if candidate.RuntimeType == RuntimeVolcHardwareAgent {
+			if _, err := tx.Exec(ctx, `UPDATE hardware_agent_session SET status='STOPPING',updated_at=$2 WHERE conversation_id=$1::uuid`, candidate.InternalID, now); err != nil {
+				return err
+			}
+		} else {
+			if _, err := tx.Exec(ctx, `UPDATE rtc_task SET status='STOPPING', lease_expires_at=NULL,
+				updated_at=$2 WHERE conversation_id=$1::uuid`, candidate.InternalID, now); err != nil {
+				return err
+			}
 		}
 		if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history (
 			conversation_id, from_status, to_status, reason, source
 		) VALUES ($1::uuid, 'ACTIVE', 'STOPPING', $2, 'worker')`, candidate.InternalID, historyReason); err != nil {
 			return err
 		}
-		if err := s.enqueueRecovery(ctx, tx, candidate, "conversation.agent_stop_requested", candidate.ProviderTaskRef, reason); err != nil {
-			return err
+		if candidate.RuntimeType != RuntimeVolcHardwareAgent {
+			if err := s.enqueueRecovery(ctx, tx, candidate, "conversation.agent_stop_requested", candidate.ProviderTaskRef, reason); err != nil {
+				return err
+			}
 		}
 	case conversation.StatusStartingAgent:
 		action = "reconcile_starting_task"
@@ -175,6 +191,22 @@ func (s *RecoveryScanner) recoverCandidate(ctx context.Context, tx pgx.Tx, candi
 			return err
 		}
 	case conversation.StatusStopping:
+		if candidate.RuntimeType == RuntimeVolcHardwareAgent {
+			action = "fail_unconfirmed_hardware_stop"
+			if _, err := tx.Exec(ctx, `UPDATE conversation SET status='FAILED',ended_at=$2,failure_code='hardware_agent_stop_timeout',version=version+1,updated_at=$2 WHERE id=$1::uuid AND status='STOPPING'`, candidate.InternalID, now); err != nil {
+				return err
+			}
+			if _, err := tx.Exec(ctx, `UPDATE hardware_agent_session SET status='FAILED',provider_ended_at=$2,updated_at=$2 WHERE conversation_id=$1::uuid`, candidate.InternalID, now); err != nil {
+				return err
+			}
+			if _, err := tx.Exec(ctx, `INSERT INTO conversation_state_history(conversation_id,from_status,to_status,reason,source) VALUES($1::uuid,'STOPPING','FAILED','hardware agent stop was not confirmed before deadline','worker')`, candidate.InternalID); err != nil {
+				return err
+			}
+			if err := enqueueHardwareContentExtraction(ctx, tx, s.outbox, candidate.TenantID, candidate.InternalID); err != nil {
+				return err
+			}
+			break
+		}
 		action = "reconcile_stopping_task"
 		deduplicationKey := candidate.ProviderTaskRef + ":" + strconv.FormatInt(recoveryBucket(now, s.options.StoppingAfter), 10)
 		reason := candidate.StopReason