From feb99de1929566f2f936e9473bba7c45297642e6 Mon Sep 17 00:00:00 2001 From: Codex Local Date: Wed, 26 Aug 2026 22:56:19 +0800 Subject: [PATCH] feat: require production infrastructure evidence --- deploy/server/README.md | 6 +- deploy/server/health-monitor.sh | 50 +++++++- deploy/server/install-monitoring.sh | 1 + docs/development-plan.md | 2 +- docs/development-status.md | 7 +- docs/device-api-compatibility.md | 7 +- docs/external-acceptance-checklist.md | 4 +- docs/platform-health-reporting.md | 7 +- docs/release-evidence-bundle.md | 4 +- docs/requirements-traceability.md | 2 +- docs/server-health-monitoring.md | 14 ++- internal/diagnostics/acceptance_snapshot.go | 109 ++++++++++++++---- .../acceptance_snapshot_integration_test.go | 5 +- .../diagnostics/acceptance_snapshot_test.go | 26 +++++ openapi/openapi.yaml | 22 +++- scripts/smoke-acceptance-snapshot.ps1 | 8 +- scripts/smoke-release-evidence.ps1 | 2 +- web/src/api/schema.d.ts | 16 ++- web/src/pages/OpsDiagnosticsWorkspace.tsx | 6 +- 19 files changed, 237 insertions(+), 61 deletions(-) diff --git a/deploy/server/README.md b/deploy/server/README.md index d30b6b9..f34abaa 100644 --- a/deploy/server/README.md +++ b/deploy/server/README.md @@ -1,6 +1,6 @@ # 阿里云单机演示部署 -该配置用于单机公网演示，不是生产验收配置。只向公网发布HTTP 80和HTTPS 443；PostgreSQL、Redis、MinIO、Keycloak内部端口以及API/Worker端口均不映射到宿主机。当前阿里云演示机使用Let's Encrypt短期IP证书，`certbot-ip-renew.timer`负责自动续期。单机备份覆盖PostgreSQL、Redis、MinIO对象和当前Keycloak数据目录，并执行数据库级隔离恢复校验；五分钟健康检查覆盖公网入口、OIDC、容器、备份、证书、定时任务和磁盘。跨故障域备份、真实告警接收器、真实RTC、模型、内容安全和云KMS仍按阶段8验收清单执行。 +该配置用于单机公网演示，不是生产验收配置。只向公网发布HTTP 80和HTTPS 443；PostgreSQL、Redis、MinIO、Keycloak内部端口以及API/Worker端口均不映射到宿主机。当前阿里云演示机使用Let's Encrypt短期IP证书，`certbot-ip-renew.timer`负责自动续期。单机备份覆盖PostgreSQL、Redis、MinIO对象和当前Keycloak数据目录，并执行数据库级隔离恢复校验；五分钟健康检查覆盖部署模式、公网入口、OIDC、容器、模拟服务、备份、外部告警、证书、定时任务和磁盘。跨故障域备份、真实告警接收器、真实RTC、模型、内容安全和云KMS仍按阶段8验收清单执行。 服务器至少需要Docker Engine 29与Compose 2.40。首次部署先生成权限为600的服务器专用配置： @@ -54,7 +54,9 @@ sudo systemctl show platform-health-monitor.service -p Result -p ExecMainStatus python3 -m json.tool /var/lib/data-terminal-cloud-monitor/status.json ``` -监测任务每五分钟执行，正常状态为`healthy`。它还要求宿主机所有非回环TCP监听只能使用22、80和443，避免安全组误放行与错误容器端口映射叠加后暴露数据库或缓存。默认只记录结构化状态和systemd日志；在`/etc/data-terminal-cloud/monitor.env`配置`ALERT_WEBHOOK_URL`后，状态变化会发送带时间戳和HMAC-SHA256签名的JSON通知。完整检查项、签名规则、阈值和失败分支见[《单机云端健康监测与告警手册》](../../docs/server-health-monitoring.md)。 +监测任务每五分钟执行，正常状态为`healthy`。默认`HOST_DEPLOYMENT_PROFILE=demo`，要求两个模拟服务完整运行；切到`production`后则要求模拟服务全部停止。它还要求宿主机所有非回环TCP监听只能使用22、80和443，避免安全组误放行与错误容器端口映射叠加后暴露数据库或缓存。默认只记录结构化状态和systemd日志；在`/etc/data-terminal-cloud/monitor.env`配置`ALERT_WEBHOOK_URL`后，状态变化会发送带时间戳和HMAC-SHA256签名的JSON通知。完整检查项、签名规则、阈值和失败分支见[《单机云端健康监测与告警手册》](../../docs/server-health-monitoring.md)。 + +生产候选不能只修改部署模式。生产验收快照同时要求：`HOST_DEPLOYMENT_PROFILE=production`、OSS异地备份最近一次上传与回读验证成功、真实外部告警接收器已配置、`mock-provider`和`mock-alert-receiver`均不运行。任一条件缺失都会阻断生产基础设施门禁，但不会影响`LAB`或`STAGING`联调。 ## OSS异地备份适配 diff --git a/deploy/server/health-monitor.sh b/deploy/server/health-monitor.sh index e899a07..e924755 100755 --- a/deploy/server/health-monitor.sh +++ b/deploy/server/health-monitor.sh @@ -21,6 +21,7 @@ alert_webhook_secret_file=${ALERT_WEBHOOK_SECRET_FILE:-/etc/data-terminal-cloud/ platform_health_report_url=${PLATFORM_HEALTH_REPORT_URL:-} platform_health_report_secret_file=${PLATFORM_HEALTH_REPORT_SECRET_FILE:-/etc/data-terminal-cloud/platform-health-callback-secret} expected_public_tcp_ports=${EXPECTED_PUBLIC_TCP_PORTS:-22,80,443} +deployment_profile=${HOST_DEPLOYMENT_PROFILE:-demo} if [[ -f "$offsite_config_file" ]]; then # shellcheck disable=SC1090 source "$offsite_config_file" @@ -41,6 +42,10 @@ if [[ ! "$expected_public_tcp_ports" =~ ^[0-9]+(,[0-9]+)*$ ]]; then echo "EXPECTED_PUBLIC_TCP_PORTS must contain comma-separated integers" >&2 exit 2 fi +if [[ "$deployment_profile" != demo && "$deployment_profile" != production ]]; then + echo "HOST_DEPLOYMENT_PROFILE must be demo or production" >&2 + exit 2 +fi compose=(docker compose -f "$project_root/compose.yaml" -f "$project_root/deploy/server/compose.override.yaml" --env-file "$project_root/deploy/server/.env") mkdir -p "$state_dir" @@ -73,6 +78,21 @@ record_check() { esac } +record_check deployment_profile healthy "$deployment_profile" +if [[ -z "$alert_webhook_url" ]]; then + if [[ "$deployment_profile" == production ]]; then + record_check external_alert_receiver critical not_configured + else + record_check external_alert_receiver healthy not_configured + fi +elif [[ ! -r "$alert_webhook_secret_file" ]]; then + record_check external_alert_receiver critical secret_missing +elif (( $(tr -d '\r\n' <"$alert_webhook_secret_file" | wc -c) < 32 )); then + record_check external_alert_receiver critical secret_invalid +else + record_check external_alert_receiver healthy configured +fi + public_origin=$(sed -n 's/^PUBLIC_ORIGIN=//p' "$project_root/deploy/server/.env") if [[ "$public_origin" == https://* ]]; then record_check public_origin healthy "$public_origin" @@ -113,7 +133,10 @@ else record_check oidc_issuer critical request_failed fi -required_services=(postgres redis minio keycloak api callbacks worker web gateway mock-provider mock-alert-receiver) +required_services=(postgres redis minio keycloak api callbacks worker web gateway) +if [[ "$deployment_profile" == demo ]]; then + required_services+=(mock-provider mock-alert-receiver) +fi for service in "${required_services[@]}"; do container=$("${compose[@]}" ps -q "$service") if [[ -z "$container" ]]; then @@ -131,6 +154,25 @@ for service in "${required_services[@]}"; do fi done +mock_services_running=() +for mock_service in mock-provider mock-alert-receiver; do + mock_container=$("${compose[@]}" ps -q "$mock_service") + if [[ -n "$mock_container" && $(docker inspect -f '{{.State.Status}}' "$mock_container") == running ]]; then + mock_services_running+=("$mock_service") + fi +done +if [[ "$deployment_profile" == production ]]; then + if (( ${#mock_services_running[@]} > 0 )); then + record_check mock_services critical unexpected_running + else + record_check mock_services healthy disabled + fi +elif (( ${#mock_services_running[@]} == 2 )); then + record_check mock_services healthy enabled_for_demo +else + record_check mock_services critical incomplete_demo_stack +fi + latest_backup=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n1 | cut -d' ' -f2- || true) if [[ -z "$latest_backup" || ! -f "$latest_backup" ]]; then record_check backup_freshness critical missing @@ -168,7 +210,11 @@ except Exception: print("invalid|0|")' "$offsite_state_file") fi fi else - record_check offsite_backup healthy not_required + if [[ "$deployment_profile" == production ]]; then + record_check offsite_backup critical not_required + else + record_check offsite_backup healthy not_required + fi fi backup_result=$(systemctl show platform-backup.service -p Result --value 2>/dev/null || true) diff --git a/deploy/server/install-monitoring.sh b/deploy/server/install-monitoring.sh index ddab760..2d8e8d4 100755 --- a/deploy/server/install-monitoring.sh +++ b/deploy/server/install-monitoring.sh @@ -32,6 +32,7 @@ if [[ ! -e /etc/data-terminal-cloud/monitor.env ]]; then report_url="$public_origin/callbacks/v1/platform-health" fi cat >/etc/data-terminal-cloud/monitor.env < 更新日期：2026-08-26 -> 当前接口契约版本：0.66.0 +> 当前接口契约版本：0.67.0 > 当前主里程碑：M8目标环境联调、试点与上线验收（外部条件待提供） ## 阶段状态 @@ -15,7 +15,7 @@ | 第五阶段：SaaS核心业务开发 | 本地业务闭环完成 | 已完成家庭成员邀请与角色协作、长者、稳定事实、五类分项授权、设备读取/配置/绑定、可靠命令、签名固件灰度/暂停/回滚和逐设备升级进度、通话查询、双轨录音状态/授权/播放下载的数据库、服务、HTTP契约与React工作台；服务端隔离、ETag、审计、纵向烟测及Chrome/Edge兼容检查均通过 | 固件生产签名设施、macOS Safari和类生产容量仍待外部环境验证 | | 第六阶段：Agent内容与记忆闭环 | 本地开发闭环完成，生产模型协议已接入 | 最终话轮、可切换本地/火山方舟内容生成器与Embedding、事务外模型调用、摘要、开放话题、故事/事实候选、人工版本审核、授权门禁、已确认记忆、索引失效、固定边界召回及React审核页面均已实现；端到端烟测通过 | 真实方舟API Key、内容效果与召回质量验收、模型评测和真实内容安全策略待完成 | | 第七阶段：生产加固 | 本地证据持续收敛 | 记忆可靠删除、隐私访问/导出/删除、运营配置灰度、用量费用额度、告警处置闭环、可重试HMAC告警Webhook及不污染正式告警的接收器联通测试、三类Grafana仪表盘、证书到期监测、OIDC/方舟/RTC外部基础设施探针、本地密钥轮换、生产配置拒绝启动门禁、全部本地专项容量、隔离恢复、供应商故障、安全扫描及Chrome/Edge兼容均通过；主要业务核心包已达到70%覆盖率，供应商客户端达到关键分支85%目标 | 全仓覆盖率及HTTP公共层仍低于需求目标；真实告警网关和值班组映射、生产等价容量、云密钥托管和Safari仍待完成 | -| 第八阶段：联调试点与上线 | 本地准备完成，外部联调待执行 | 外部依赖预检、联调问题门禁、候选验收快照、阻断告警定位、接收器联通测试、内容安全TLS预检和按业务时区生成的试点日/周报均已形成可执行入口；本地真实OIDC、Worker、HMAC验签接收、六项技术门禁及日/周报下载通过 | 远程CI、真实投诉证据、告警网关和值班组映射、真实供应商/真机联调、家庭试点、预生产演练和生产发布 | +| 第八阶段：联调试点与上线 | 本地准备完成，外部联调待执行 | 外部依赖预检、联调问题门禁、候选验收快照、阻断告警定位、接收器联通测试、内容安全TLS预检和按业务时区生成的试点日/周报均已形成可执行入口；本地真实OIDC、Worker、HMAC验签接收、八项技术门禁及日/周报下载通过；生产快照不能再用演示模式、未启用异地备份、未配置外部告警或仍运行Mock服务冒充可上线 | 远程CI、真实投诉证据、告警网关和值班组映射、真实供应商/真机联调、家庭试点、预生产演练和生产发布 | ## 本轮验证记录 @@ -181,7 +181,7 @@ | 隐私删除并发 | 100个并发受理全部成功、HTTP失败率0%、提交p95 385.15ms；11.58秒内600个子任务、100个墓碑和100个主体冻结全部对账 | | PostgreSQL隔离恢复 | 实际`pg_dump`/`pg_restore`到随机隔离库，迁移`22|false`、66张表、190个外键和关键计数一致；本地约5.14秒，临时库/备份已清理 | | 阿里云单机加密备份与恢复 | 目标服务器已安装每日systemd备份任务，覆盖PostgreSQL、认证Redis RDB、MinIO全部桶和暂停后的Keycloak数据目录；归档使用AES-256-CBC/PBKDF2加密、逐文件SHA-256并保留14天。首次实际归档成功，数据库从V38备份恢复到随机隔离库后`38|false|2|3|73|215`六项证据一致，归档内Redis RDB格式有效、Keycloak 3个文件一致、临时恢复库为0；同机备份尚不作为跨故障域生产恢复证据 | -| 阿里云单机健康监测 | 目标服务器已安装五分钟systemd健康检查，结构化状态覆盖HTTPS、宿主机公网监听端口、OIDC签发地址、11个常驻容器、本机/异地备份状态与摘要、备份执行结果、三个运行timer、短期IP证书剩余时间和磁盘阈值。监听策略只允许非回环TCP 22/80/443，回环数据库端口不误报，额外端口会进入critical。实际正常路径23项全部healthy；把本机备份最大年龄临时设为0秒后仅`backup_freshness`进入critical并退出1。可选Webhook仅在状态变化时发送带时间戳和HMAC-SHA256的脱敏状态；真实接收器尚未配置，因此不作为值班通知送达证据 | +| 阿里云单机健康监测 | 目标服务器已安装五分钟systemd健康检查，结构化状态覆盖部署模式、外部告警配置、HTTPS、宿主机公网监听端口、OIDC签发地址、核心容器、模拟服务、本机/异地备份状态与摘要、备份执行结果、三个运行timer、短期IP证书剩余时间和磁盘阈值。监听策略只允许非回环TCP 22/80/443，回环数据库端口不误报，额外端口会进入critical。演示正常路径26项全部healthy；生产模式要求模拟服务退出。可选Webhook仅在状态变化时发送带时间戳和HMAC-SHA256的脱敏状态；真实接收器尚未配置，因此不作为值班通知送达证据 | | 阿里云OSS异地备份适配 | 目标服务器已按官方SHA-256安装ossutil 2.3.0，新增每日OSS同步任务、加密归档及摘要上传、远端回读复算、原子状态和健康强制门禁。ECS正式路径现使用IMDSv2和`EcsRamRole`自动短期凭据，不保存长期AK；隔离配置模式验证上传/回读2个文件并得到`verified`，目标实例缺角色时退出4并记录`failed/ecs_ram_role_missing`。2026-08-26实测地域`cn-hangzhou`且角色元数据404；当前仍无RAM角色和Bucket，正式任务保持`disabled/not_enabled`且健康状态为`not_required`，不能作为真实异地备份证据 | | 进程与Redis恢复 | API/Callbacks/Worker重启恢复；停机期间Outbox仅执行一次；Redis 14个热状态键清空后由签名设备请求重建，PostgreSQL权威计数不变 | | 供应商故障恢复 | V23持久化按租户/供应商的可用状态；连续临时失败暂停新启动但停止任务继续尝试；演练确认暂停后新外呼0、停止尝试2次、历史接口200、CRITICAL告警出现，恢复后全部收敛；V23单版本回退/重升通过 | @@ -251,6 +251,7 @@ | 托管内容安全外部预检与告警 | OpenAPI 0.64.0把`content_safety_tls`纳入外部依赖查询和候选验收快照。生产Worker仅执行DNS、TCP和受信TLS握手，不发送待分类正文；结果复用依赖可用性指标、数据库脱敏状态和运营诊断页面，连续5分钟不可用进入统一CRITICAL告警，记录缺失/失败/过期均可阻断候选技术快照但不驱逐Pod。按低强度策略执行App/探针/诊断定向测试、前端类型生成/类型检查/生产构建和Prometheus规则校验。 | | 云端主机健康状态接入 | OpenAPI 0.65.0新增独立HMAC签名的主机健康回调、30天脱敏快照表和运营只读状态页面。五分钟周期上报覆盖公网入口、容器、备份、证书、磁盘和定时器；无签名、过期、重复检查及综合状态不一致报告被拒绝，页面超过15分钟未更新明确标记过期。该能力不替代整机、Callbacks或数据库失联时必须使用的外部探针。 | | 云端主机健康发布门禁 | OpenAPI 0.66.0把最新主机健康快照纳入发布候选技术验收：无上报、超过15分钟、综合预警或严重异常均阻断，只有当前有效且23项检查全部正常才通过，其中包括宿主机公网监听端口策略。快照仅归档状态、时间和检查数量，不复制磁盘、证书、监听地址或备份明细值。 | +| 生产基础设施强制门禁 | OpenAPI 0.67.0把生产基础设施状态作为第八项技术门禁。`PRODUCTION`快照必须从当前主机健康报告同时证明`production`部署模式、OSS异地备份上传回读校验不过期、外部告警接收器已配置及两个Mock服务均已退出；演示机继续明确上报`demo`，因此生产快照会被如实阻断。`LAB/STAGING`标记为不适用，不影响开发联调。前端同步展示部署模式、告警和Mock状态；该门禁只证明配置与当前进程状态，不替代真实告警送达演练和生产基础设施一致性报告。 | | 阶段8外部执行轻量预检 | 新增`scripts/stage8-preflight.ps1`，以单一只读入口核对候选Git提交、远程仓库、Compose本地基线、外部配置变量名称和十类证据索引结构/状态；报告明确不读取密钥值，也不把环境变量或本地服务可用性当作验收证据。默认仅报告缺口，`-Strict`可供自动化阻断未就绪交接；当前已建立本地`main`候选提交，本地运行基线就绪且索引结构有效，仍缺远程仓库，样例构建摘要为占位值，十类外部证据均为`PENDING`。 | | 候选构建身份贯通 | Compose新增统一候选构建参数，后端二进制和后端/Web镜像OCI标签共享版本、完整Git提交及UTC构建时间；远程CI从OpenAPI版本和运行编号形成候选版本，构建后重新读取两个镜像标签并与`GITHUB_SHA`和构建时间逐项比对，再把候选版本、提交、时间和双镜像摘要写入构建元数据。修复了此前Dockerfile虽支持注入但Compose/CI未传参、运行版本仍为`dev/unknown`的问题。 | diff --git a/docs/device-api-compatibility.md b/docs/device-api-compatibility.md index 4191aaf..b787abe 100644 --- a/docs/device-api-compatibility.md +++ b/docs/device-api-compatibility.md @@ -1,7 +1,7 @@ # 设备 API 兼容性约定 > 当前设备接口主版本：`/device/v2` -> 当前接口契约：OpenAPI 0.66.0 +> 当前接口契约：OpenAPI 0.67.0 > 适用对象：云端 API、终端固件、模拟终端和设备联调工具 ## 1. 兼容目标 @@ -42,9 +42,10 @@ CI在拉取请求中以目标分支OpenAPI为基线执行破坏性变更检查 | OpenAPI版本 | 设备兼容变更 | 兼容结论 | |---|---|---| -| 0.64.0 | 运营外部预检增加`content_safety_tls`枚举和候选快照第四类可选必需依赖；未修改设备端点、字段、状态机或终端行为 | v2完全兼容 | -| 0.65.0 | 新增主机健康签名回调和运营只读查询；未修改设备端点、字段、状态机或终端行为 | v2完全兼容 | +| 0.67.0 | 发布候选快照增加生产基础设施强制门禁及主机部署模式、异地备份、外部告警、模拟服务状态；未修改设备端点、字段、状态机或终端行为 | v2完全兼容 | | 0.66.0 | 发布候选技术验收快照增加主机健康当前性门禁；未修改设备端点、字段、状态机或终端行为 | v2完全兼容 | +| 0.65.0 | 新增主机健康签名回调和运营只读查询；未修改设备端点、字段、状态机或终端行为 | v2完全兼容 | +| 0.64.0 | 运营外部预检增加`content_safety_tls`枚举和候选快照第四类可选必需依赖；未修改设备端点、字段、状态机或终端行为 | v2完全兼容 | | 0.63.0 | 增加设备固件阶段事件查询及云端证据字段；设备已有命令回执仅记录同一结果的不可变事件，未增加旧终端必填字段 | v2兼容，旧终端行为不变 | | 0.62.0 | 命令结果增加A/B写入、待重启、启动健康和回退可选字段，省略时保持原有单分区结果语义 | v2兼容，旧终端可继续省略 | | 0.61.0 | 新增家庭侧固件版本影响只读接口，未修改设备协议 | v2完全兼容 | diff --git a/docs/external-acceptance-checklist.md b/docs/external-acceptance-checklist.md index 3964036..19cb140 100644 --- a/docs/external-acceptance-checklist.md +++ b/docs/external-acceptance-checklist.md @@ -58,7 +58,7 @@ powershell -ExecutionPolicy Bypass -File scripts/stage8-preflight.ps1 ` - 发现的问题均进入V36联调问题台账并具备严重级别、根因、修复版本和脱敏回归证据；清单全量汇总显示未关闭HIGH/CRITICAL为0； - 真实凭据完成一次轮换，旧凭据失效且业务不中断。 - 使用`cmd/kms-rotation-conformance`归档同一候选构建的云提供器、工作负载身份、动态读取和滚动重启两类轮换；报告必须为`PASS/REAL_KMS_HSM`，证据包内`kms_rotation_validation.coverage_complete=true`。 -- 以相同候选构建重新生成验收快照并归档，六项技术门禁均通过；若候选构建变化，原快照作废并重新执行受影响场景。 +- 以相同候选构建重新生成验收快照并归档，八项技术门禁均通过；若候选构建变化，原快照作废并重新执行受影响场景。 ## 3. 家庭试点 @@ -97,7 +97,7 @@ powershell -ExecutionPolicy Bypass -File scripts/stage8-preflight.ps1 ` - 最近两个Chrome/Edge和macOS最近两个Safari版本通过关键流程； - 使用`cmd/safari-conformance`归档同一候选在真实Safari `CURRENT/PREVIOUS`、两视口和六流程的24项结果；报告必须为`PASS/REAL_SAFARI`，证据包内`safari_compatibility_validation.coverage_complete=true`。 - 核心模块覆盖率和关键分支覆盖率达到需求规定门槛，当前本地未达标项必须在发布前关闭或由风险负责人书面接受。 -- 已使用生产候选镜像摘要生成`PRODUCTION_RELEASE`验收快照；运行版本必须等于候选版本且包含提交号和构建时间，六项技术门禁全部通过。 +- 已使用生产候选镜像摘要生成`PRODUCTION_RELEASE`验收快照；运行版本必须等于候选版本且包含提交号和构建时间，八项技术门禁全部通过，其中生产基础设施门禁必须证明生产部署模式、已验证异地备份、真实外部告警配置和Mock服务退出。 - 使用`cmd/release-evidence`为同一候选版本、镜像摘要和源提交生成最终证据包；十类证据索引不得存在`PENDING`。出现`WAIVED`时必须附有权责任人的书面风险接受，包状态只能进入带豁免的发布评审，不能自动视为批准；八类不可豁免门禁不得使用`WAIVED`。 ### 4.2 业务与合规门禁 diff --git a/docs/platform-health-reporting.md b/docs/platform-health-reporting.md index 1f9bb3c..bb9a6af 100644 --- a/docs/platform-health-reporting.md +++ b/docs/platform-health-reporting.md @@ -2,7 +2,7 @@ ## 1. 目标与边界 -单机健康监测原先只把结果保存在主机`/var/lib/data-terminal-cloud-monitor/status.json`。本功能把同一份脱敏状态每五分钟签名上报到Callbacks进程，并在运营“联调诊断”工作台展示最新结果，覆盖公网HTTPS、OIDC签发地址、核心容器、本地及异地备份、证书、磁盘和systemd定时器。 +单机健康监测原先只把结果保存在主机`/var/lib/data-terminal-cloud-monitor/status.json`。本功能把同一份脱敏状态每五分钟签名上报到Callbacks进程，并在运营“联调诊断”工作台展示最新结果，覆盖部署模式、公网HTTPS、OIDC签发地址、核心容器、模拟服务、本地及异地备份、外部告警配置、证书、磁盘和systemd定时器。 该链路用于当前状态查看和排障，不替代外部告警网关：Callbacks、数据库或整机同时不可用时，平台无法把自身故障写入自身数据库，仍需阿里云云监控、独立探针或真实值班网关从系统外发现。 @@ -35,7 +35,7 @@ | `POST /callbacks/v1/platform-health` | 独立HMAC签名 | 新快照返回201；完全重复返回200 | | `GET /api/v1/ops/diagnostics/platform-health` | OIDC + 运营角色 + 客户组织选择 | 返回全平台最新脱敏快照 | -接口契约见`openapi/openapi.yaml` 0.66.0。最新快照同时进入发布候选技术验收门禁：缺失、超过15分钟、warning或critical都会阻断，只有当前healthy快照通过。 +接口契约见`openapi/openapi.yaml` 0.67.0。最新快照同时进入发布候选技术验收门禁：缺失、超过15分钟、warning或critical都会阻断，只有当前healthy快照通过。`PRODUCTION`快照还执行独立的生产基础设施门禁：主机必须明确上报`production`部署模式，异地备份必须已完成远端回读校验且不过期，外部告警接收器必须配置，两个模拟服务必须停止。`LAB`和`STAGING`保留该门禁字段但标记为不适用，不影响现有开发与联调。 ## 5. 单机配置 @@ -44,8 +44,11 @@ ```dotenv PLATFORM_HEALTH_REPORT_URL=https://121.41.46.238/callbacks/v1/platform-health PLATFORM_HEALTH_REPORT_SECRET_FILE=/etc/data-terminal-cloud/platform-health-callback-secret +HOST_DEPLOYMENT_PROFILE=demo ``` +不得仅把`HOST_DEPLOYMENT_PROFILE`改为`production`来制造放行结果。切换前应先停用模拟服务，启用并验证OSS异地备份，配置真实外部告警接收器及HMAC密钥，再执行一次主机健康检查并重新生成生产快照。 + 启用后执行： ```bash diff --git a/docs/release-evidence-bundle.md b/docs/release-evidence-bundle.md index e6c38c3..0265685 100644 --- a/docs/release-evidence-bundle.md +++ b/docs/release-evidence-bundle.md @@ -4,7 +4,7 @@ `cmd/release-evidence`用于阶段8联调、试点和生产发布前，把同一候选版本的技术验收快照、外部验收状态、真机固件一致性报告、接口契约、数据库迁移、部署清单、关键交付文档和可用SBOM封装为一个带SHA-256清单的ZIP。`PILOT`包还必须携带同一客户组织、同一业务时区、上一完整自然周的7份日报和1份周报。它解决证据散落、候选身份不一致、真机场景缺口被模拟结果掩盖、试点周期证据缺失以及归档后无法判断文件是否被替换的问题。 -证据包不是发布批准。生成器只在技术验收快照六项门禁全部通过时工作，但外部证据可以保持`PENDING`，此时包内状态固定为`EXTERNAL_VALIDATION_PENDING`。真实设备、供应商、KMS、告警接收器、Safari或业务签署不能用本地模拟结果替代。 +证据包不是发布批准。生成器只在技术验收快照八项门禁全部通过时工作；其中生产快照必须额外证明生产部署模式、已验证异地备份、外部告警配置和模拟服务退出。外部证据可以保持`PENDING`，此时包内状态固定为`EXTERNAL_VALIDATION_PENDING`。真实设备、供应商、KMS、告警接收器、Safari或业务签署不能用本地模拟结果替代。 ## 2. 输入设计 @@ -143,4 +143,4 @@ go test ./internal/releaseevidence ./cmd/release-evidence .\scripts\smoke-release-evidence.ps1 ``` -烟测使用真实OIDC生成绑定同一候选版本和API镜像摘要的`PILOT`技术验收快照、上一完整自然周的7份日报和1份周报，构造十项全部`PENDING`的外部索引，生成并解包临时ZIP，核对OpenAPI 0.66.0、v3清单、候选一致性、周期证据、摘要、待验计数和真机固件四场景缺口，随后清理临时文件、审计和幂等记录。技术验收快照还要求主机健康上报在15分钟内且没有warning/critical。该结果只证明打包链路可用，不证明十项外部验收、真机固件或投诉证据已经完成。 +烟测使用真实OIDC生成绑定同一候选版本和API镜像摘要的`PILOT`技术验收快照、上一完整自然周的7份日报和1份周报，构造十项全部`PENDING`的外部索引，生成并解包临时ZIP，核对OpenAPI 0.67.0、v3清单、候选一致性、周期证据、摘要、待验计数和真机固件四场景缺口，随后清理临时文件、审计和幂等记录。技术验收快照还要求主机健康上报在15分钟内且没有warning/critical；生产类型另要求生产基础设施门禁通过。该结果只证明打包链路可用，不证明十项外部验收、真机固件或投诉证据已经完成。 diff --git a/docs/requirements-traceability.md b/docs/requirements-traceability.md index 7c87daa..e8ef595 100644 --- a/docs/requirements-traceability.md +++ b/docs/requirements-traceability.md @@ -87,7 +87,7 @@ | 来源 | 要求 | 当前处理 | |---|---|---| -| P0414 | 首期需求、验收用例、缺陷和风险形成对应关系 | 本文完成首期功能到代码、接口、数据和验证的基线；V30问题清单由V36补充严重级别、根因、修复版本、回归证据及未关闭HIGH/CRITICAL发布阻断汇总；OpenAPI 0.50.0—0.66.0可按候选构建导出七项技术门禁及脱敏阻断告警定位快照，绑定候选版本/构建摘要后按业务时区归档日/周运行证据。第七项门禁要求主机健康快照在15分钟内且全部检查正常。`cmd/release-evidence`现使用v3清单，把同一候选的快照、接口、迁移、部署、SBOM和十类外部状态形成逐文件摘要；远程CI供应链、真机固件、真实终端RTC、真实Agent模型、真实告警网关、云KMS/HSM轮换、生产等价恢复、生产基础设施和Safari兼容分别要求结构化报告，并把14项CI检查/镜像/SBOM、设备固件场景、语音方式与恢复场景、生成/Embedding/内容安全质量、路由演练、轮换模式、恢复类型、基础设施控制项及Safari矩阵覆盖与缺口写入清单。V38及九个离线校验器把云端状态与远程构建、外部设备、RTC/语音供应商、模型提供器、网关、密钥提供器、目标环境和浏览器证据绑定；`content_safety_tls`把策略服务网络/证书状态纳入指标、告警、运营页面和候选快照。缺检查、缺制品、缺场景、缺演练、缺质量类别、缺控制项、`PENDING`和本地模拟证据均不会被误报为外部验收完成，整体发布仍必须基于真实联调、灰度和审批记录确认。 | +| P0414 | 首期需求、验收用例、缺陷和风险形成对应关系 | 本文完成首期功能到代码、接口、数据和验证的基线；V30问题清单由V36补充严重级别、根因、修复版本、回归证据及未关闭HIGH/CRITICAL发布阻断汇总；OpenAPI 0.50.0—0.67.0可按候选构建导出八项技术门禁及脱敏阻断告警定位快照，绑定候选版本/构建摘要后按业务时区归档日/周运行证据。第七项门禁要求主机健康快照在15分钟内且全部检查正常；第八项在`PRODUCTION`强制要求生产部署模式、已验证异地备份、外部告警配置及Mock服务退出，在`LAB/STAGING`明确标记不适用。`cmd/release-evidence`现使用v3清单，把同一候选的快照、接口、迁移、部署、SBOM和十类外部状态形成逐文件摘要；远程CI供应链、真机固件、真实终端RTC、真实Agent模型、真实告警网关、云KMS/HSM轮换、生产等价恢复、生产基础设施和Safari兼容分别要求结构化报告，并把14项CI检查/镜像/SBOM、设备固件场景、语音方式与恢复场景、生成/Embedding/内容安全质量、路由演练、轮换模式、恢复类型、基础设施控制项及Safari矩阵覆盖与缺口写入清单。V38及九个离线校验器把云端状态与远程构建、外部设备、RTC/语音供应商、模型提供器、网关、密钥提供器、目标环境和浏览器证据绑定；`content_safety_tls`把策略服务网络/证书状态纳入指标、告警、运营页面和候选快照。缺检查、缺制品、缺场景、缺演练、缺质量类别、缺控制项、`PENDING`和本地模拟证据均不会被误报为外部验收完成，整体发布仍必须基于真实联调、灰度和审批记录确认。 | | P0445 | 需求编号关联代码、测试、接口或配置变更 | 本文逐项建立关联；后续功能变更必须同步更新本矩阵和`development-status.md`。 | 当前仍不能宣告完成的部分集中为：真实火山RTC与豆包三模式联调、真实方舟内容与Embedding质量、真实听筒的长者交流体验、量产设备/KMS注入、跨故障域生产对象存储与备份、真机异常恢复、真实账单和生产级内容安全。阿里云单机环境已经形成加密全组件归档、数据库隔离恢复和Redis RDB校验，但同机归档不能替代托管PITR、异地副本和生产恢复演练。这些剩余项属于阶段8外部验收，不应通过增加本地低价值测试伪造完成证据。 diff --git a/docs/server-health-monitoring.md b/docs/server-health-monitoring.md index 673568f..0c25ad2 100644 --- a/docs/server-health-monitoring.md +++ b/docs/server-health-monitoring.md @@ -2,7 +2,7 @@ ## 1. 目标与边界 -`deploy/server/health-monitor.sh`为单机阿里云演示环境提供五分钟一次的轻量健康检查。它用于发现“公网入口不可用、宿主机出现未授权公网监听端口、身份签发地址漂移、容器退出、备份过期或损坏、证书接近到期、定时任务停止、磁盘空间不足”等直接影响可用性或暴露面的事件。 +`deploy/server/health-monitor.sh`为单机阿里云环境提供五分钟一次的轻量健康检查。它用于发现“部署模式与运行服务不一致、公网入口不可用、宿主机出现未授权公网监听端口、身份签发地址漂移、容器退出、备份过期或损坏、外部告警未配置、证书接近到期、定时任务停止、磁盘空间不足”等直接影响可用性或暴露面的事件。 该脚本不替代Prometheus、Grafana、云监控或真实值班系统。默认只写结构化状态文件和systemd日志；配置外部接收器后，状态发生变化时才发送一次HMAC签名Webhook，避免每五分钟重复通知。 @@ -10,12 +10,15 @@ | 检查 | 正常条件 | 失败状态 | |---|---|---| +| 部署模式 | `HOST_DEPLOYMENT_PROFILE`只能为`demo`或`production` | 非法值直接拒绝执行 | +| 外部告警接收器 | `demo`未配置时明确记录`not_configured`；`production`必须配置且HMAC密钥不少于32字节 | `not_configured`、`secret_missing`或`secret_invalid` | | 公网地址 | `PUBLIC_ORIGIN`为HTTPS | `invalid_https_origin` | | HTTPS入口 | `/api/v1/version`在10秒内返回成功 | `request_failed` | | 公网监听端口 | 宿主机非回环TCP监听端口只能属于`22,80,443`；回环地址上的数据库或监控端口不计入公网暴露 | `unexpected_ports=...`、`probe_unavailable`或`invalid_probe_result` | | OIDC | Discovery中的`issuer`与公网地址完全一致 | `issuer_mismatch`或`request_failed` | -| 容器 | PostgreSQL、Redis、MinIO、Keycloak、API、Callbacks、Worker、Web、Gateway和两个模拟服务均为运行态；有健康检查的容器必须为healthy | `missing`、容器状态或健康状态 | -| 备份 | 最新加密归档不超过36小时且外层SHA-256有效 | `missing`、`age_seconds=...`或`checksum_failed` | +| 容器 | PostgreSQL、Redis、MinIO、Keycloak、API、Callbacks、Worker、Web和Gateway均为运行态；有健康检查的容器必须为healthy | `missing`、容器状态或健康状态 | +| 模拟服务 | `demo`要求模拟语音和模拟告警服务都运行；`production`要求两者都不运行 | `incomplete_demo_stack`或`unexpected_running` | +| 备份 | 最新加密归档不超过36小时且外层SHA-256有效；`production`还要求异地备份启用并完成最新归档的回读验证 | `missing`、`age_seconds=...`、`checksum_failed`或异地备份状态错误 | | 备份任务 | 最近一次`platform-backup.service`结果成功且退出码为0 | `result=.../exit=...` | | 定时任务 | 备份和证书续期timer均为enabled/active | 当前启用/活动状态 | | 证书 | 本地活动证书剩余时间不少于36小时 | `missing`或`remaining_seconds=...` | @@ -86,6 +89,7 @@ PLATFORM_HEALTH_REPORT_SECRET_FILE=/etc/data-terminal-cloud/platform-health-call | 环境变量 | 默认值 | 说明 | |---|---|---| +| `HOST_DEPLOYMENT_PROFILE` | `demo` | 主机部署模式；生产切换必须与异地备份、告警接收器和模拟服务退出同时完成 | | `BACKUP_MAX_AGE_SECONDS` | `129600` | 最新备份最大年龄，36小时 | | `CERTIFICATE_MIN_SECONDS` | `129600` | 证书最小剩余时间，36小时 | | `DISK_WARNING_PERCENT` | `80` | 磁盘警告阈值 | @@ -130,9 +134,11 @@ systemctl list-timers platform-health-monitor.timer platform-backup.timer certbo docker compose -f compose.yaml -f deploy/server/compose.override.yaml --env-file deploy/server/.env ps -a ``` +生产分支可在不改变线上容器的情况下验证。临时配置`HOST_DEPLOYMENT_PROFILE=production`运行检查时，只要模拟服务仍在运行，结果必须为critical且`mock_services=unexpected_running`；测试状态目录应与正式目录隔离。 + ## 8. 下一步生产化 1. 配置真实告警网关、主值班组和升级组，并完成HMAC验签联通测试。 2. 将状态接入云监控或Prometheus长期存储，建立可用率和告警送达趋势。 -3. 将加密备份复制到不同故障域，并对“异地复制最后成功时间”增加检查。 +3. 启用现有OSS异地备份任务，确认上传、回读和SHA-256复算成功，并把最近验证时间纳入健康状态。 4. 使用外部OIDC、托管数据库/Redis/对象存储后，把资源级高可用、PITR和容量指标纳入同一状态模型。 diff --git a/internal/diagnostics/acceptance_snapshot.go b/internal/diagnostics/acceptance_snapshot.go index 5f056c9..298ea83 100644 --- a/internal/diagnostics/acceptance_snapshot.go +++ b/internal/diagnostics/acceptance_snapshot.go @@ -82,6 +82,15 @@ type AcceptancePlatformHealthState struct { GateSatisfied bool `json:"gate_satisfied"` } +type AcceptanceProductionInfrastructureState struct { + Applicable bool `json:"applicable"` + DeploymentProfile string `json:"deployment_profile"` + OffsiteBackupReady bool `json:"offsite_backup_ready"` + ExternalAlertReceiverReady bool `json:"external_alert_receiver_ready"` + MockServicesDisabled bool `json:"mock_services_disabled"` + GateSatisfied bool `json:"gate_satisfied"` +} + type AcceptanceBlockingAlert struct { ID string `json:"id"` RuleCode string `json:"rule_code"` @@ -101,25 +110,26 @@ type AcceptanceGate struct { } type AcceptanceSnapshot struct { - ID string `json:"id"` - GeneratedAt time.Time `json:"generated_at"` - TenantID string `json:"tenant_id"` - CandidateVersion string `json:"candidate_version"` - BuildDigest string `json:"build_digest"` - Environment string `json:"environment"` - RunType string `json:"run_type"` - Scope string `json:"scope"` - RedactionPolicy string `json:"redaction_policy"` - RuntimeBuild buildinfo.Info `json:"runtime_build"` - Database AcceptanceDatabaseState `json:"database"` - IntegrationIssues IntegrationIssueSummary `json:"integration_issues"` - ExternalDependencies []AcceptanceDependencyState `json:"external_dependencies"` - Outbox AcceptanceOutboxState `json:"outbox"` - Alerts AcceptanceAlertState `json:"alerts"` - PlatformHealth AcceptancePlatformHealthState `json:"platform_health"` - Gates []AcceptanceGate `json:"gates"` - OverallGateSatisfied bool `json:"overall_gate_satisfied"` - Disclaimer string `json:"disclaimer"` + ID string `json:"id"` + GeneratedAt time.Time `json:"generated_at"` + TenantID string `json:"tenant_id"` + CandidateVersion string `json:"candidate_version"` + BuildDigest string `json:"build_digest"` + Environment string `json:"environment"` + RunType string `json:"run_type"` + Scope string `json:"scope"` + RedactionPolicy string `json:"redaction_policy"` + RuntimeBuild buildinfo.Info `json:"runtime_build"` + Database AcceptanceDatabaseState `json:"database"` + IntegrationIssues IntegrationIssueSummary `json:"integration_issues"` + ExternalDependencies []AcceptanceDependencyState `json:"external_dependencies"` + Outbox AcceptanceOutboxState `json:"outbox"` + Alerts AcceptanceAlertState `json:"alerts"` + PlatformHealth AcceptancePlatformHealthState `json:"platform_health"` + ProductionInfrastructure AcceptanceProductionInfrastructureState `json:"production_infrastructure"` + Gates []AcceptanceGate `json:"gates"` + OverallGateSatisfied bool `json:"overall_gate_satisfied"` + Disclaimer string `json:"disclaimer"` } func (s *Service) ExportAcceptanceSnapshot(ctx context.Context, principal identity.Principal, tenantPublicID, key, requestID string, input AcceptanceSnapshotInput) (AcceptanceSnapshot, bool, error) { @@ -289,11 +299,12 @@ func buildAcceptanceSnapshot(ctx context.Context, tx pgx.Tx, tenantPublicID, ten ); err != nil { return AcceptanceSnapshot{}, err } - platformHealth, err := loadAcceptancePlatformHealth(ctx, tx, now) + platformHealth, productionInfrastructure, err := loadAcceptancePlatformHealth(ctx, tx, now, input.Environment) if err != nil { return AcceptanceSnapshot{}, err } snapshot.PlatformHealth = platformHealth + snapshot.ProductionInfrastructure = productionInfrastructure runtimeSatisfied := input.Environment == "LAB" || (snapshot.RuntimeBuild.Version == input.CandidateVersion && snapshot.RuntimeBuild.Commit != "unknown" && snapshot.RuntimeBuild.BuildTime != "unknown") dependenciesSatisfied := true @@ -310,6 +321,7 @@ func buildAcceptanceSnapshot(ctx context.Context, tx pgx.Tx, tenantPublicID, ten {Code: "outbox_clear", Satisfied: outboxSatisfied, Summary: "租户及平台级待处理、处理中和死信任务必须清零"}, {Code: "critical_alerts_clear", Satisfied: alertsSatisfied, Summary: "不得存在未解决的高或严重级告警"}, {Code: "platform_health_current", Satisfied: snapshot.PlatformHealth.GateSatisfied, Summary: acceptancePlatformHealthSummary(snapshot.PlatformHealth)}, + {Code: "production_infrastructure_ready", Satisfied: snapshot.ProductionInfrastructure.GateSatisfied, Summary: acceptanceProductionInfrastructureSummary(snapshot.ProductionInfrastructure)}, } snapshot.OverallGateSatisfied = true for _, gate := range snapshot.Gates { @@ -318,7 +330,7 @@ func buildAcceptanceSnapshot(ctx context.Context, tx pgx.Tx, tenantPublicID, ten return snapshot, nil } -func loadAcceptancePlatformHealth(ctx context.Context, tx pgx.Tx, now time.Time) (AcceptancePlatformHealthState, error) { +func loadAcceptancePlatformHealth(ctx context.Context, tx pgx.Tx, now time.Time, environment string) (AcceptancePlatformHealthState, AcceptanceProductionInfrastructureState, error) { state := AcceptancePlatformHealthState{State: "MISSING"} var observedAt, receivedAt time.Time var checksJSON []byte @@ -328,16 +340,16 @@ func loadAcceptancePlatformHealth(ctx context.Context, tx pgx.Tx, now time.Time) &state.Status, &observedAt, &receivedAt, &checksJSON, ) if errors.Is(err, pgx.ErrNoRows) { - return state, nil + return state, evaluateAcceptanceProductionInfrastructure(environment, nil), nil } if err != nil { - return AcceptancePlatformHealthState{}, err + return AcceptancePlatformHealthState{}, AcceptanceProductionInfrastructureState{}, err } var checks []PlatformHealthCheck if err = json.Unmarshal(checksJSON, &checks); err != nil { - return AcceptancePlatformHealthState{}, err + return AcceptancePlatformHealthState{}, AcceptanceProductionInfrastructureState{}, err } - return evaluateAcceptancePlatformHealth(state.Status, observedAt, receivedAt, checks, now), nil + return evaluateAcceptancePlatformHealth(state.Status, observedAt, receivedAt, checks, now), evaluateAcceptanceProductionInfrastructure(environment, checks), nil } func evaluateAcceptancePlatformHealth(status string, observedAt, receivedAt time.Time, checks []PlatformHealthCheck, now time.Time) AcceptancePlatformHealthState { @@ -372,6 +384,53 @@ func acceptancePlatformHealthSummary(state AcceptancePlatformHealthState) string return fmt.Sprintf("云端主机健康快照当前有效，共%d项检查", state.CheckCount) } +func evaluateAcceptanceProductionInfrastructure(environment string, checks []PlatformHealthCheck) AcceptanceProductionInfrastructureState { + state := AcceptanceProductionInfrastructureState{Applicable: environment == "PRODUCTION", DeploymentProfile: "MISSING"} + if !state.Applicable { + state.GateSatisfied = true + return state + } + profileReady := false + for _, check := range checks { + switch check.Name { + case "deployment_profile": + state.DeploymentProfile = check.Value + profileReady = check.Status == "healthy" && check.Value == "production" + case "offsite_backup": + state.OffsiteBackupReady = check.Status == "healthy" && strings.HasPrefix(check.Value, "age_seconds=") + case "external_alert_receiver": + state.ExternalAlertReceiverReady = check.Status == "healthy" && check.Value == "configured" + case "mock_services": + state.MockServicesDisabled = check.Status == "healthy" && check.Value == "disabled" + } + } + state.GateSatisfied = profileReady && state.OffsiteBackupReady && state.ExternalAlertReceiverReady && state.MockServicesDisabled + return state +} + +func acceptanceProductionInfrastructureSummary(state AcceptanceProductionInfrastructureState) string { + if !state.Applicable { + return "非生产候选不适用生产基础设施强制门禁" + } + missing := make([]string, 0, 4) + if state.DeploymentProfile != "production" { + missing = append(missing, "生产部署模式") + } + if !state.OffsiteBackupReady { + missing = append(missing, "已验证异地备份") + } + if !state.ExternalAlertReceiverReady { + missing = append(missing, "真实外部告警接收器") + } + if !state.MockServicesDisabled { + missing = append(missing, "关闭模拟服务") + } + if len(missing) > 0 { + return "生产基础设施尚缺：" + strings.Join(missing, "、") + } + return "生产部署模式、异地备份、外部告警和模拟服务退出均已确认" +} + func acceptanceRuntimeSummary(environment string) string { if environment == "LAB" { return "实验室环境允许开发构建，但候选版本与镜像摘要必须明确" diff --git a/internal/diagnostics/acceptance_snapshot_integration_test.go b/internal/diagnostics/acceptance_snapshot_integration_test.go index ed2f439..0d9e551 100644 --- a/internal/diagnostics/acceptance_snapshot_integration_test.go +++ b/internal/diagnostics/acceptance_snapshot_integration_test.go @@ -59,7 +59,7 @@ func TestAcceptanceSnapshotIsAuditedIdempotentAndTenantScoped(t *testing.T) { if err != nil || !created { t.Fatalf("first=%+v created=%t err=%v", first, created, err) } - if first.ID == "" || first.Database.MigrationVersion < 39 || first.Database.Dirty || first.RedactionPolicy != "ops-acceptance-minimal-v1" || len(first.ExternalDependencies) != 1 || len(first.Gates) != 7 { + if first.ID == "" || first.Database.MigrationVersion < 39 || first.Database.Dirty || first.RedactionPolicy != "ops-acceptance-minimal-v1" || len(first.ExternalDependencies) != 1 || len(first.Gates) != 8 { t.Fatalf("unexpected snapshot=%+v", first) } if first.ExternalDependencies[0].Dependency != "oidc_jwks" || first.ExternalDependencies[0].ServiceName != "api" { @@ -77,6 +77,9 @@ func TestAcceptanceSnapshotIsAuditedIdempotentAndTenantScoped(t *testing.T) { if first.Gates[6].Code != "platform_health_current" || first.PlatformHealth.State == "" { t.Fatalf("platform health=%+v gates=%+v", first.PlatformHealth, first.Gates) } + if first.Gates[7].Code != "production_infrastructure_ready" || !first.Gates[7].Satisfied || first.ProductionInfrastructure.Applicable { + t.Fatalf("production infrastructure=%+v gates=%+v", first.ProductionInfrastructure, first.Gates) + } replayed, createdAgain, err := service.ExportAcceptanceSnapshot(ctx, admin, tenantPublicID, key, requestID, input) if err != nil || createdAgain || replayed.ID != first.ID || !replayed.GeneratedAt.Equal(first.GeneratedAt) { t.Fatalf("replayed=%+v created=%t err=%v", replayed, createdAgain, err) diff --git a/internal/diagnostics/acceptance_snapshot_test.go b/internal/diagnostics/acceptance_snapshot_test.go index 2470a79..5403ea9 100644 --- a/internal/diagnostics/acceptance_snapshot_test.go +++ b/internal/diagnostics/acceptance_snapshot_test.go @@ -71,6 +71,32 @@ func TestAcceptancePlatformHealthSummaryCoversMissingAndCounts(t *testing.T) { } } +func TestEvaluateAcceptanceProductionInfrastructure(t *testing.T) { + readyChecks := []PlatformHealthCheck{ + {Name: "deployment_profile", Status: "healthy", Value: "production"}, + {Name: "offsite_backup", Status: "healthy", Value: "age_seconds=120"}, + {Name: "external_alert_receiver", Status: "healthy", Value: "configured"}, + {Name: "mock_services", Status: "healthy", Value: "disabled"}, + } + ready := evaluateAcceptanceProductionInfrastructure("PRODUCTION", readyChecks) + if !ready.Applicable || !ready.GateSatisfied || !ready.OffsiteBackupReady || !ready.ExternalAlertReceiverReady || !ready.MockServicesDisabled { + t.Fatalf("ready=%+v", ready) + } + demo := evaluateAcceptanceProductionInfrastructure("PRODUCTION", []PlatformHealthCheck{ + {Name: "deployment_profile", Status: "healthy", Value: "demo"}, + {Name: "offsite_backup", Status: "healthy", Value: "not_required"}, + {Name: "external_alert_receiver", Status: "healthy", Value: "not_configured"}, + {Name: "mock_services", Status: "healthy", Value: "enabled_for_demo"}, + }) + if demo.GateSatisfied || demo.DeploymentProfile != "demo" || !strings.Contains(acceptanceProductionInfrastructureSummary(demo), "异地备份") { + t.Fatalf("demo=%+v summary=%q", demo, acceptanceProductionInfrastructureSummary(demo)) + } + lab := evaluateAcceptanceProductionInfrastructure("LAB", nil) + if lab.Applicable || !lab.GateSatisfied { + t.Fatalf("lab=%+v", lab) + } +} + func TestAcceptanceSnapshotAllowsAllProductionExternalDependencies(t *testing.T) { input := AcceptanceSnapshotInput{ CandidateVersion: "0.64.0-rc.1", BuildDigest: "sha256:" + strings.Repeat("b", 64), diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 1d5bfd7..419adb1 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -1,8 +1,8 @@ openapi: 3.1.0 info: title: 数据采集终端—云端系统 API - version: 0.66.0 - description: 工程框架、SaaS核心业务、设备v2能力协商与智能体启动状态查询、首次通话就绪与最低固件门禁、背景资料准备失败闭环、活动通话授权撤回安全停止、语音内容闭环、终端终稿补传、批量设备遥测、录音上传队列、隐私对账、固件、双轨录音、状态事件、具体资源分享、受控运营配置、Agent组合组件引用、质量证据门禁、运营自定义版本族投放、新通话运行时冻结与供应商启动传递、带并发版本校验的配置投放暂停恢复、带严重度和验收证据门禁的联调问题闭环、含阻断告警定位及当前主机健康证据的发布候选技术验收快照、绑定候选版本与构建摘要并按业务时区生成的可审计试点日/周运行证据、可审计且不生成正式告警的接收器联通测试、外部依赖预检台账、证书到期清单，以及带价格版本冻结、供应商用量去重和硬额度控制的费用治理接口。 + version: 0.67.0 + description: 工程框架、SaaS核心业务、设备v2能力协商与智能体启动状态查询、首次通话就绪与最低固件门禁、背景资料准备失败闭环、活动通话授权撤回安全停止、语音内容闭环、终端终稿补传、批量设备遥测、录音上传队列、隐私对账、固件、双轨录音、状态事件、具体资源分享、受控运营配置、Agent组合组件引用、质量证据门禁、运营自定义版本族投放、新通话运行时冻结与供应商启动传递、带并发版本校验的配置投放暂停恢复、带严重度和验收证据门禁的联调问题闭环、含阻断告警定位、当前主机健康及生产基础设施强制条件的发布候选技术验收快照、绑定候选版本与构建摘要并按业务时区生成的可审计试点日/周运行证据、可审计且不生成正式告警的接收器联通测试、外部依赖预检台账、证书到期清单，以及带价格版本冻结、供应商用量去重和硬额度控制的费用治理接口。 servers: - url: http://localhost:58080 description: 本地开发 @@ -3284,6 +3284,17 @@ components: warning_count: { type: integer, minimum: 0, maximum: 64 } critical_count: { type: integer, minimum: 0, maximum: 64 } gate_satisfied: { type: boolean, description: 仅当快照未超过15分钟且综合状态及全部检查均为healthy时为true。 } + OpsAcceptanceProductionInfrastructureState: + type: object + additionalProperties: false + required: [applicable, deployment_profile, offsite_backup_ready, external_alert_receiver_ready, mock_services_disabled, gate_satisfied] + properties: + applicable: { type: boolean, description: 目标环境为PRODUCTION时为true。 } + deployment_profile: { type: string, enum: [MISSING, demo, production] } + offsite_backup_ready: { type: boolean, description: 最近主机报告证明异地备份已启用、已回读验证且未过期。 } + external_alert_receiver_ready: { type: boolean, description: 最近主机报告证明真实外部告警接收器已配置且密钥可读。 } + mock_services_disabled: { type: boolean, description: 最近主机报告证明模拟语音与模拟告警服务均未运行。 } + gate_satisfied: { type: boolean } OpsAcceptanceDependencyState: type: object additionalProperties: false @@ -3338,13 +3349,13 @@ components: additionalProperties: false required: [code, satisfied, summary] properties: - code: { type: string, enum: [candidate_build_identified, database_migration_clean, required_dependencies_available, integration_issues_clear, outbox_clear, critical_alerts_clear, platform_health_current] } + code: { type: string, enum: [candidate_build_identified, database_migration_clean, required_dependencies_available, integration_issues_clear, outbox_clear, critical_alerts_clear, platform_health_current, production_infrastructure_ready] } satisfied: { type: boolean } summary: { type: string } OpsAcceptanceSnapshot: type: object additionalProperties: false - required: [id, generated_at, tenant_id, candidate_version, build_digest, environment, run_type, scope, redaction_policy, runtime_build, database, integration_issues, external_dependencies, outbox, alerts, platform_health, gates, overall_gate_satisfied, disclaimer] + required: [id, generated_at, tenant_id, candidate_version, build_digest, environment, run_type, scope, redaction_policy, runtime_build, database, integration_issues, external_dependencies, outbox, alerts, platform_health, production_infrastructure, gates, overall_gate_satisfied, disclaimer] properties: id: { type: string, format: uuid } generated_at: { type: string, format: date-time } @@ -3362,7 +3373,8 @@ components: outbox: { $ref: '#/components/schemas/OpsAcceptanceOutboxState' } alerts: { $ref: '#/components/schemas/OpsAcceptanceAlertState' } platform_health: { $ref: '#/components/schemas/OpsAcceptancePlatformHealthState' } - gates: { type: array, minItems: 7, maxItems: 7, items: { $ref: '#/components/schemas/OpsAcceptanceGate' } } + production_infrastructure: { $ref: '#/components/schemas/OpsAcceptanceProductionInfrastructureState' } + gates: { type: array, minItems: 8, maxItems: 8, items: { $ref: '#/components/schemas/OpsAcceptanceGate' } } overall_gate_satisfied: { type: boolean } disclaimer: { type: string } OpsAcceptanceSnapshotResponse: diff --git a/scripts/smoke-acceptance-snapshot.ps1 b/scripts/smoke-acceptance-snapshot.ps1 index eab0109..69ea281 100644 --- a/scripts/smoke-acceptance-snapshot.ps1 +++ b/scripts/smoke-acceptance-snapshot.ps1 @@ -37,9 +37,11 @@ try { if ($createdResponse.StatusCode -ne 201 -or $snapshot.redaction_policy -ne 'ops-acceptance-minimal-v1') { throw '验收快照首次生成结果不正确' } if ($snapshot.database.migration_version -lt 36 -or $snapshot.database.dirty) { throw '验收快照未正确冻结数据库迁移状态' } if (@($snapshot.external_dependencies).Count -ne 1 -or $snapshot.external_dependencies[0].dependency -ne 'oidc_jwks' -or -not $snapshot.external_dependencies[0].gate_satisfied) { throw '验收快照未正确冻结OIDC外部依赖状态' } - if (@($snapshot.gates).Count -ne 6) { throw '验收快照门禁数量不正确' } + if (@($snapshot.gates).Count -ne 8) { throw '验收快照门禁数量不正确' } + $productionGate = @($snapshot.gates | Where-Object { $_.code -eq 'production_infrastructure_ready' }) + if ($productionGate.Count -ne 1 -or -not $productionGate[0].satisfied -or $snapshot.production_infrastructure.applicable -or -not $snapshot.production_infrastructure.gate_satisfied) { throw 'LAB快照不应被生产基础设施门禁阻断' } $failedGates = @($snapshot.gates | Where-Object { -not $_.satisfied }) - if ($snapshot.overall_gate_satisfied -ne ($failedGates.Count -eq 0)) { throw '验收快照总结果与六项门禁不一致' } + if ($snapshot.overall_gate_satisfied -ne ($failedGates.Count -eq 0)) { throw '验收快照总结果与八项门禁不一致' } $blockingItems = @($snapshot.alerts.blocking_items) $expectedItems = [Math]::Min([int]$snapshot.alerts.blocking, 100) if ($blockingItems.Count -ne $expectedItems -or $snapshot.alerts.blocking_items_truncated -ne ($snapshot.alerts.blocking -gt 100)) { throw '阻断告警定位数量或截断标识不正确' } @@ -60,7 +62,7 @@ try { if ($null -eq $_.Exception.Response -or [int]$_.Exception.Response.StatusCode -ne 403) { throw } } $failedCodes = if ($failedGates.Count) { ($failedGates.code -join ',') } else { 'none' } - Write-Host "发布候选验收快照烟测通过：真实OIDC、迁移冻结、外部依赖、六项门禁、阻断告警定位、幂等重放、租户隔离和端点脱敏均正确；当前阻断项=$failedCodes。" + Write-Host "发布候选验收快照烟测通过：真实OIDC、迁移冻结、外部依赖、八项门禁、生产条件按环境适用、阻断告警定位、幂等重放、租户隔离和端点脱敏均正确；当前阻断项=$failedCodes。" } finally { docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -c "BEGIN; DELETE FROM audit_event WHERE request_id='$requestID'; DELETE FROM idempotency_record WHERE idempotency_key='$key'; COMMIT;" | Out-Null Write-Host '已清理验收快照烟测审计和幂等记录；快照本身不在服务端留存。' diff --git a/scripts/smoke-release-evidence.ps1 b/scripts/smoke-release-evidence.ps1 index 073305a..51d184d 100644 --- a/scripts/smoke-release-evidence.ps1 +++ b/scripts/smoke-release-evidence.ps1 @@ -78,7 +78,7 @@ try { if ($result.status -ne 'EXTERNAL_VALIDATION_PENDING' -or $result.pending -ne 10 -or $result.verified -ne 0 -or $result.sha256 -notmatch '^sha256:[a-f0-9]{64}$') { throw '证据包生成结果不正确' } Expand-Archive -LiteralPath $bundlePath -DestinationPath $expandedPath $manifest = Get-Content -Raw -LiteralPath (Join-Path $expandedPath 'release-manifest.json') | ConvertFrom-Json - if (-not $manifest.technical_gate_satisfied -or $manifest.evidence_status -ne 'EXTERNAL_VALIDATION_PENDING' -or $manifest.pending_count -ne 10 -or $manifest.openapi_version -ne '0.66.0' -or $manifest.schema_version -ne 'release-candidate-evidence-bundle-v3' -or -not $manifest.pilot_evidence_included -or $manifest.pilot_daily_count -ne 7 -or $manifest.pilot_week_start -ne $weekStartText -or $manifest.pilot_timezone -ne 'Asia/Shanghai') { throw '证据包清单内容不正确' } + if (-not $manifest.technical_gate_satisfied -or $manifest.evidence_status -ne 'EXTERNAL_VALIDATION_PENDING' -or $manifest.pending_count -ne 10 -or $manifest.openapi_version -ne '0.67.0' -or $manifest.schema_version -ne 'release-candidate-evidence-bundle-v3' -or -not $manifest.pilot_evidence_included -or $manifest.pilot_daily_count -ne 7 -or $manifest.pilot_week_start -ne $weekStartText -or $manifest.pilot_timezone -ne 'Asia/Shanghai') { throw '证据包清单内容不正确' } if ($manifest.firmware_validation.status -ne 'PENDING' -or $manifest.firmware_validation.coverage_complete -or $manifest.firmware_validation.required_device_count -ne 3 -or @($manifest.firmware_validation.required_scenarios).Count -ne 4 -or @($manifest.firmware_validation.gaps).Count -ne 5) { throw '真机固件场景缺口未在证据包中保留' } if ($manifest.alert_receiver_validation.status -ne 'PENDING' -or $manifest.alert_receiver_validation.coverage_complete -or $manifest.alert_receiver_validation.report_present -or @($manifest.alert_receiver_validation.gaps).Count -ne 2) { throw '真实告警接收器与正式演练缺口未在证据包中保留' } if ($manifest.kms_rotation_validation.status -ne 'PENDING' -or $manifest.kms_rotation_validation.coverage_complete -or $manifest.kms_rotation_validation.report_present -or @($manifest.kms_rotation_validation.gaps).Count -ne 3) { throw '真实KMS/HSM两类轮换缺口未在证据包中保留' } diff --git a/web/src/api/schema.d.ts b/web/src/api/schema.d.ts index 7e6bfeb..552f083 100644 --- a/web/src/api/schema.d.ts +++ b/web/src/api/schema.d.ts @@ -2709,6 +2709,19 @@ export interface components { /** @description 仅当快照未超过15分钟且综合状态及全部检查均为healthy时为true。 */ gate_satisfied: boolean; }; + OpsAcceptanceProductionInfrastructureState: { + /** @description 目标环境为PRODUCTION时为true。 */ + applicable: boolean; + /** @enum {string} */ + deployment_profile: "MISSING" | "demo" | "production"; + /** @description 最近主机报告证明异地备份已启用、已回读验证且未过期。 */ + offsite_backup_ready: boolean; + /** @description 最近主机报告证明真实外部告警接收器已配置且密钥可读。 */ + external_alert_receiver_ready: boolean; + /** @description 最近主机报告证明模拟语音与模拟告警服务均未运行。 */ + mock_services_disabled: boolean; + gate_satisfied: boolean; + }; OpsAcceptanceDependencyState: { /** @enum {string} */ service_name: "api" | "worker"; @@ -2773,7 +2786,7 @@ export interface components { }; OpsAcceptanceGate: { /** @enum {string} */ - code: "candidate_build_identified" | "database_migration_clean" | "required_dependencies_available" | "integration_issues_clear" | "outbox_clear" | "critical_alerts_clear" | "platform_health_current"; + code: "candidate_build_identified" | "database_migration_clean" | "required_dependencies_available" | "integration_issues_clear" | "outbox_clear" | "critical_alerts_clear" | "platform_health_current" | "production_infrastructure_ready"; satisfied: boolean; summary: string; }; @@ -2800,6 +2813,7 @@ export interface components { outbox: components["schemas"]["OpsAcceptanceOutboxState"]; alerts: components["schemas"]["OpsAcceptanceAlertState"]; platform_health: components["schemas"]["OpsAcceptancePlatformHealthState"]; + production_infrastructure: components["schemas"]["OpsAcceptanceProductionInfrastructureState"]; gates: components["schemas"]["OpsAcceptanceGate"][]; overall_gate_satisfied: boolean; disclaimer: string; diff --git a/web/src/pages/OpsDiagnosticsWorkspace.tsx b/web/src/pages/OpsDiagnosticsWorkspace.tsx index 1ddf643..10e2304 100644 --- a/web/src/pages/OpsDiagnosticsWorkspace.tsx +++ b/web/src/pages/OpsDiagnosticsWorkspace.tsx @@ -149,8 +149,8 @@ function serviceLabel(value:OpsExternalDependency["service_name"]){return ({api: function externalDependencyState(item:OpsExternalDependency){const stale=Date.now()-new Date(item.checked_at).getTime()>item.probe_interval_ms*3;return stale?{label:"结果过期",tone:"warning",stale:true}:item.status==="AVAILABLE"?{label:"可用",tone:"active",stale:false}:{label:"不可用",tone:"danger",stale:false}} function platformHealthState(item:PlatformHealthSnapshot){const stale=Date.now()-new Date(item.observed_at).getTime()>15*60*1000;if(stale)return{label:"上报过期",tone:"danger",stale:true};return item.status==="healthy"?{label:"运行正常",tone:"active",stale:false}:item.status==="warning"?{label:"存在预警",tone:"warning",stale:false}:{label:"严重异常",tone:"danger",stale:false}} function platformHealthCheckStatusLabel(value:PlatformHealthSnapshot["checks"][number]["status"]){return value==="healthy"?"正常":value==="warning"?"预警":"严重"} -function platformHealthCheckLabel(value:string){const labels:Record={public_origin:"公网地址",public_https:"HTTPS入口",public_tcp_listeners:"公网监听端口",oidc_issuer:"身份签发地址",backup_freshness:"本地备份",offsite_backup:"异地备份",backup_service:"备份服务","platform-backup.timer":"备份定时器","platform-offsite-backup.timer":"异地备份定时器","certbot-ip-renew.timer":"证书续期定时器",certificate:"HTTPS证书",disk_usage:"磁盘空间",container_postgres:"PostgreSQL",container_redis:"Redis",container_minio:"对象存储",container_keycloak:"身份服务",container_api:"业务API",container_callbacks:"回调服务",container_worker:"后台任务",container_web:"前端页面",container_gateway:"HTTPS网关","container_mock-provider":"模拟语音提供器","container_mock-alert-receiver":"模拟告警接收器"};return labels[value]??value} -function platformHealthValueLabel(_name:string,value:string){if(value==="not_required")return "当前未要求启用";if(value==="issuer_matches")return "签发地址一致";if(value==="http_200")return "公网访问成功";if(value==="enabled/active")return "已启用并运行";if(value==="success")return "最近执行成功";if(value.startsWith("ports="))return `当前对外端口 ${value.slice(6)}`;if(value.startsWith("unexpected_ports="))return `发现未授权端口 ${value.slice(17)}`;if(value.startsWith("age_seconds=")){const seconds=Number(value.slice(12));return Number.isFinite(seconds)?`距最近成功 ${formatDuration(seconds*1000)}`:value}if(value.startsWith("remaining_seconds=")){const seconds=Number(value.slice(18));return Number.isFinite(seconds)?`剩余 ${Math.floor(seconds/86400)} 天` : value}if(value.startsWith("used_percent="))return `已使用 ${value.slice(13)}%`;if(value==="running/healthy"||value==="running/none")return "容器正在运行";return value} +function platformHealthCheckLabel(value:string){const labels:Record={deployment_profile:"部署模式",external_alert_receiver:"外部告警接收器",mock_services:"模拟服务",public_origin:"公网地址",public_https:"HTTPS入口",public_tcp_listeners:"公网监听端口",oidc_issuer:"身份签发地址",backup_freshness:"本地备份",offsite_backup:"异地备份",backup_service:"备份服务","platform-backup.timer":"备份定时器","platform-offsite-backup.timer":"异地备份定时器","certbot-ip-renew.timer":"证书续期定时器",certificate:"HTTPS证书",disk_usage:"磁盘空间",container_postgres:"PostgreSQL",container_redis:"Redis",container_minio:"对象存储",container_keycloak:"身份服务",container_api:"业务API",container_callbacks:"回调服务",container_worker:"后台任务",container_web:"前端页面",container_gateway:"HTTPS网关","container_mock-provider":"模拟语音提供器","container_mock-alert-receiver":"模拟告警接收器"};return labels[value]??value} +function platformHealthValueLabel(_name:string,value:string){if(value==="demo")return "演示环境";if(value==="production")return "生产环境";if(value==="not_configured")return "尚未配置";if(value==="configured")return "已配置";if(value==="enabled_for_demo")return "演示模式已启用";if(value==="disabled")return "已关闭";if(value==="unexpected_running")return "生产环境仍在运行模拟服务";if(value==="not_required")return "当前未要求启用";if(value==="issuer_matches")return "签发地址一致";if(value==="http_200")return "公网访问成功";if(value==="enabled/active")return "已启用并运行";if(value==="success")return "最近执行成功";if(value.startsWith("ports="))return `当前对外端口 ${value.slice(6)}`;if(value.startsWith("unexpected_ports="))return `发现未授权端口 ${value.slice(17)}`;if(value.startsWith("age_seconds=")){const seconds=Number(value.slice(12));return Number.isFinite(seconds)?`距最近成功 ${formatDuration(seconds*1000)}`:value}if(value.startsWith("remaining_seconds=")){const seconds=Number(value.slice(18));return Number.isFinite(seconds)?`剩余 ${Math.floor(seconds/86400)} 天` : value}if(value.startsWith("used_percent="))return `已使用 ${value.slice(13)}%`;if(value==="running/healthy"||value==="running/none")return "容器正在运行";return value} function downloadJSON(value:unknown,filename:string){const blob=new Blob([JSON.stringify(value,null,2)],{type:"application/json"});const url=URL.createObjectURL(blob);const anchor=document.createElement("a");anchor.href=url;anchor.download=filename;anchor.click();URL.revokeObjectURL(url)} -function acceptanceGateLabel(value:OpsAcceptanceSnapshot["gates"][number]["code"]){return ({candidate_build_identified:"候选构建",database_migration_clean:"数据库迁移",required_dependencies_available:"必需外部依赖",integration_issues_clear:"高严重度问题",outbox_clear:"可靠任务",critical_alerts_clear:"高严重度告警",platform_health_current:"云端主机健康"} as Record)[value]} +function acceptanceGateLabel(value:OpsAcceptanceSnapshot["gates"][number]["code"]){return ({candidate_build_identified:"候选构建",database_migration_clean:"数据库迁移",required_dependencies_available:"必需外部依赖",integration_issues_clear:"高严重度问题",outbox_clear:"可靠任务",critical_alerts_clear:"高严重度告警",platform_health_current:"云端主机健康",production_infrastructure_ready:"生产基础设施"} as Record)[value]} function acceptanceResourceStateLabel(value:OpsAcceptanceSnapshot["alerts"]["blocking_items"][number]["resource_state"]){return ({PRESENT:"资源存在",MISSING:"资源已不存在，仍需人工复核关闭",AGGREGATE:"聚合队列",UNKNOWN:"未提供自动核对规则"} as Record)[value]}