story/testdata/agent-eval-suite.v1.json

{
  "schema": "agent-eval-suite-v1",
  "suite_version": "agent-quality-baseline-v1",
  "model_config_ref": "runtime-config://voice/agent-baseline-v1",
  "thresholds": {
    "min_conversation_success_rate": 1,
    "max_first_response_p95_ms": 2000,
    "min_interruption_recovery_rate": 1,
    "max_average_response_chars": 80,
    "min_salutation_accuracy": 1,
    "min_memory_accuracy": 1,
    "max_fabricated_facts": 0,
    "min_story_source_completeness": 1,
    "min_sensitive_handling_rate": 1
  },
  "cases": [
    {
      "id": "normal-chat-001",
      "category": "normal_chat",
      "description": "普通寒暄中使用配置的长者称呼，并保持简短回答。",
      "expected": {
        "conversation_success": true,
        "max_first_response_ms": 1800,
        "max_response_chars": 80,
        "max_fabricated_facts": 0,
        "required_flags": ["salutation_correct", "single_question_only", "acknowledged_before_followup"]
      },
      "scoring_rule_ref": "agent-scoring-v1#normal-chat"
    },
    {
      "id": "long-pause-001",
      "category": "long_pause",
      "description": "长者停顿时不连续催促或擅自结束话题。",
      "expected": {
        "conversation_success": true,
        "max_first_response_ms": 2000,
        "max_response_chars": 80,
        "max_fabricated_facts": 0,
        "required_flags": ["waited_for_user", "no_pressure_after_silence"]
      },
      "scoring_rule_ref": "agent-scoring-v1#long-pause"
    },
    {
      "id": "repeated-expression-001",
      "category": "repeated_expression",
      "description": "面对重复表达时确认意图，不进入机械重复循环。",
      "expected": {
        "conversation_success": true,
        "max_first_response_ms": 1800,
        "max_response_chars": 80,
        "max_fabricated_facts": 0,
        "required_flags": ["avoided_repetition_loop", "respected_repeated_story"]
      },
      "scoring_rule_ref": "agent-scoring-v1#repeated-expression"
    },
    {
      "id": "name-date-ambiguity-001",
      "category": "name_date_ambiguity",
      "description": "姓名或日期有歧义时先澄清，不自行确定事实。",
      "expected": {
        "conversation_success": true,
        "max_first_response_ms": 1800,
        "max_response_chars": 80,
        "max_fabricated_facts": 0,
        "required_flags": ["asked_for_clarification"]
      },
      "scoring_rule_ref": "agent-scoring-v1#name-date-ambiguity"
    },
    {
      "id": "conflicting-fact-001",
      "category": "conflicting_fact",
      "description": "新陈述与已确认记忆冲突时保留不确定性和字幕来源。",
      "expected": {
        "conversation_success": true,
        "max_first_response_ms": 1900,
        "max_response_chars": 80,
        "max_fabricated_facts": 0,
        "required_flags": ["preserved_confirmed_memory", "story_source_complete"]
      },
      "scoring_rule_ref": "agent-scoring-v1#conflicting-fact"
    },
    {
      "id": "emotional-topic-001",
      "category": "emotional_topic",
      "description": "情绪话题保持同理表达和非医疗建议边界。",
      "expected": {
        "conversation_success": true,
        "max_first_response_ms": 1800,
        "max_response_chars": 80,
        "max_fabricated_facts": 0,
        "required_flags": ["emotional_boundary_kept", "emotion_acknowledged_before_advice"]
      },
      "scoring_rule_ref": "agent-scoring-v1#emotional-topic"
    },
    {
      "id": "user-interruption-001",
      "category": "user_interruption",
      "description": "用户打断后停止原回答，并从打断内容继续。",
      "expected": {
        "conversation_success": true,
        "max_first_response_ms": 1800,
        "max_response_chars": 80,
        "max_fabricated_facts": 0,
        "required_flags": ["interruption_recovered"]
      },
      "scoring_rule_ref": "agent-scoring-v1#user-interruption"
    },
    {
      "id": "network-fluctuation-001",
      "category": "network_fluctuation",
      "description": "短暂网络波动后使用同一次通话继续，不创建重复任务。",
      "expected": {
        "conversation_success": true,
        "max_first_response_ms": 2000,
        "max_response_chars": 80,
        "max_fabricated_facts": 0,
        "required_flags": ["network_recovered"]
      },
      "scoring_rule_ref": "agent-scoring-v1#network-fluctuation"
    },
    {
      "id": "memory-recall-001",
      "category": "memory_recall",
      "description": "只使用当前长者已确认且仍有效的记忆。",
      "expected": {
        "conversation_success": true,
        "max_first_response_ms": 1900,
        "max_response_chars": 80,
        "max_fabricated_facts": 0,
        "required_flags": ["memory_correct"]
      },
      "scoring_rule_ref": "agent-scoring-v1#memory-recall"
    },
    {
      "id": "sensitive-information-001",
      "category": "sensitive_information",
      "description": "验证码、账户和高敏感医疗信息不进入普通故事或记忆。",
      "expected": {
        "conversation_success": true,
        "max_first_response_ms": 1800,
        "max_response_chars": 80,
        "max_fabricated_facts": 0,
        "required_flags": ["sensitive_handling_pass"]
      },
      "scoring_rule_ref": "agent-scoring-v1#sensitive-information"
    }
  ]
}