feat(routing): add route acceptance matrix scripts

docs/ROUTE_ACCEPTANCE_MATRIX.md

@@ -0,0 +1,165 @@
+# Route Acceptance Matrix
+
+日期：2026-05-29
+
+## 目标
+
+把 Phase 2 已经验证过的三类能力，收敛成固定入口和固定证据格式：
+
+1. 控制面
+2. 运行态 / 健康视图
+3. 数据面
+
+当前统一入口：
+
+- `scripts/acceptance/verify_route_control_plane.sh`
+- `scripts/acceptance/verify_route_health_ui.sh`
+- `scripts/acceptance/verify_route_data_plane.sh`
+- `scripts/acceptance/verify_route_acceptance_matrix.sh`
+
+其中 `verify_route_acceptance_matrix.sh` 是总入口，会按顺序执行前三个脚本并汇总 `summary.json`。
+
+## 验收矩阵
+
+### 1. 控制面
+
+脚本：
+
+- `scripts/acceptance/verify_route_control_plane.sh`
+
+验证范围：
+
+- 创建 `logical_group`
+- 创建 `logical_group_model`
+- 创建 `route`
+- 创建 `route_model`
+- `GET /api/logical-groups/{group_id}`
+- `PUT /api/logical-groups/{group_id}`
+- `PUT /api/logical-groups/{group_id}/routes/{route_id}`
+- `GET /api/logical-groups/{group_id}/routes`
+- `GET /api/logical-groups/{group_id}/routes/{route_id}/models`
+
+最小输入：
+
+- `CRM_BASE`
+- `CRM_ADMIN_TOKEN`
+
+主要产物：
+
+- `01-create-group.json`
+- `03-create-route.json`
+- `06-update-group.json`
+- `07-update-route.json`
+- `10-summary.json`
+
+### 2. 健康视图 / 运行态
+
+脚本：
+
+- `scripts/acceptance/verify_route_health_ui.sh`
+
+验证范围：
+
+- 公网 `route-health.html` 页面可达
+- `POST /api/routing/sticky/cooldowns`
+- `POST /api/routing/sticky/route-failures`
+- `GET /api/routing/routes/health`
+- `POST /api/routing/resolve`
+- `GET /api/routing/logs/failovers`
+
+最小输入：
+
+- `CRM_BASE`
+- `CRM_ADMIN_TOKEN`
+- 可选 `ROUTE_HEALTH_PAGE_URL`
+
+主要产物：
+
+- `00-route-health.html`
+- `08-health-before.json`
+- `09-resolve.json`
+- `10-health-after.json`
+- `11-failovers.json`
+- `12-summary.json`
+
+### 3. 数据面
+
+脚本：
+
+- `scripts/acceptance/verify_route_data_plane.sh`
+
+验证范围：
+
+- 创建临时 `logical_group / route / route_model`
+- `POST /api/routing/chat/completions`
+- `GET /api/routing/logs/decisions`
+
+最小输入：
+
+- `CRM_BASE`
+- `CRM_ADMIN_TOKEN`
+- `SHADOW_HOST_ID`
+- `SHADOW_GROUP_ID`
+- 二选一：
+  - `SUBSCRIPTION_USER_ID`
+  - `GATEWAY_API_KEY`
+
+补充说明：
+
+- 若传 `SUBSCRIPTION_USER_ID`，期望 `effective_gateway_key_source=managed_subscription`
+- 若传 `GATEWAY_API_KEY`，脚本仍可跑通，但这不覆盖“自动供给 managed key”场景
+
+主要产物：
+
+- `05-route-chat.json`
+- `06-decision-logs.json`
+- `07-summary.json`
+
+## 总入口
+
+脚本：
+
+- `scripts/acceptance/verify_route_acceptance_matrix.sh`
+
+它会创建：
+
+```text
+artifacts/phase2-routing-matrix/<timestamp>_route_matrix/
+  control_plane/
+  health_ui/
+  data_plane/
+  summary.json
+```
+
+`summary.json` 只保留最核心的摘要字段：
+
+- `control_plane_group_id`
+- `health_ui_group_id`
+- `data_plane_group_id`
+- `data_plane_request_id`
+- `data_plane_upstream_status`
+- `health_ui_resolve_route_id`
+
+## remote43 推荐执行方式
+
+如果要避免把真实 `CRM_ADMIN_TOKEN` 带回本地，推荐直接在 `remote43` 上执行：
+
+```bash
+cd /home/ubuntu/sub2api-kimi-patched-auto2-20260525_18169
+set -a
+. ./.env.crm
+set +a
+
+cd /home/ubuntu/sub2api-cn-relay-manager-git-current
+CRM_BASE="https://sub.tksea.top/portal-admin-api" \
+SHADOW_HOST_ID="<real-shadow-host-id>" \
+SHADOW_GROUP_ID="<real-shadow-group-id>" \
+SUBSCRIPTION_USER_ID="<managed-user-id>" \
+bash ./scripts/acceptance/verify_route_acceptance_matrix.sh
+```
+
+## 当前边界
+
+- 脚本只负责留证据和判断通过/失败，不负责清理临时 `logical_group`
+- 数据面脚本当前仍依赖调用方提供真实 `shadow_host_id / shadow_group_id`
+- 尚未覆盖“同公开模型双线路主备”的未来策略矩阵

scripts/acceptance/route_acceptance_lib.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+ROUTE_MATRIX_ROOT="${ROUTE_MATRIX_ROOT:-$ROOT_DIR/artifacts/phase2-routing-matrix}"
+
+require_var() {
+  local name="$1"
+  if [[ -z "${!name:-}" ]]; then
+    echo "missing required env: $name" >&2
+    exit 1
+  fi
+}
+
+ensure_artifact_dir() {
+  mkdir -p "$ARTIFACT_DIR"
+}
+
+save_json() {
+  local name="$1"
+  local payload="$2"
+  ensure_artifact_dir
+  printf '%s\n' "$payload" > "$ARTIFACT_DIR/$name.json"
+}
+
+save_text() {
+  local name="$1"
+  local payload="$2"
+  ensure_artifact_dir
+  printf '%s\n' "$payload" > "$ARTIFACT_DIR/$name"
+}
+
+json_get_file() {
+  local file="$1"
+  local key="$2"
+  python3 - "$file" "$key" <<'PY'
+import json
+import sys
+
+file_path, key = sys.argv[1:3]
+with open(file_path, 'r', encoding='utf-8') as fh:
+    value = json.load(fh)
+for part in key.split('.'):
+    if isinstance(value, dict):
+        value = value.get(part)
+    else:
+        value = None
+        break
+if value is None:
+    raise SystemExit(2)
+if isinstance(value, (dict, list)):
+    print(json.dumps(value, ensure_ascii=False))
+else:
+    print(value)
+PY
+}
+
+crm_auth_init() {
+  require_var CRM_BASE
+
+  if [[ -n "${CRM_ADMIN_TOKEN:-}" ]]; then
+    crm_token="$CRM_ADMIN_TOKEN"
+    crm_cookie_jar=""
+    return 0
+  fi
+
+  if [[ -n "${CRM_COOKIE_JAR:-}" ]]; then
+    crm_cookie_jar="$CRM_COOKIE_JAR"
+  else
+    crm_cookie_jar="$(mktemp /tmp/route-matrix-cookie.XXXXXX.jar)"
+  fi
+  rm -f "$crm_cookie_jar"
+
+  require_var CRM_ADMIN_USERNAME
+  require_var CRM_ADMIN_PASSWORD
+
+  local login_payload
+  login_payload="$(python3 - "$CRM_ADMIN_USERNAME" "$CRM_ADMIN_PASSWORD" <<'PY'
+import json
+import sys
+username, password = sys.argv[1:3]
+print(json.dumps({"username": username, "password": password}, ensure_ascii=False))
+PY
+)"
+
+  curl -fsS \
+    -c "$crm_cookie_jar" \
+    -b "$crm_cookie_jar" \
+    -H 'Content-Type: application/json' \
+    -X POST \
+    "${CRM_BASE%/}/api/admin/session/login" \
+    -d "$login_payload" > /dev/null
+  crm_token=""
+}
+
+crm_curl_json() {
+  local method="$1"
+  local path="$2"
+  local payload="${3:-}"
+  local -a curl_args
+  curl_args=(-fsS -X "$method")
+  if [[ -n "${crm_token:-}" ]]; then
+    curl_args+=(-H "Authorization: Bearer $crm_token")
+  elif [[ -n "${crm_cookie_jar:-}" ]]; then
+    curl_args+=(-b "$crm_cookie_jar" -c "$crm_cookie_jar")
+  else
+    echo "missing CRM auth: set CRM_ADMIN_TOKEN or CRM_ADMIN_USERNAME/CRM_ADMIN_PASSWORD" >&2
+    exit 2
+  fi
+  if [[ -n "$payload" ]]; then
+    curl_args+=(
+      -H 'Content-Type: application/json'
+      "${CRM_BASE%/}${path}"
+      -d "$payload"
+    )
+  else
+    curl_args+=("${CRM_BASE%/}${path}")
+  fi
+  curl "${curl_args[@]}"
+}
+
+curl_status_to_file() {
+  local url="$1"
+  local file="$2"
+  ensure_artifact_dir
+  curl -fsS "$url" -o "$file"
+}
+
+timestamp_token() {
+  date +%s
+}

scripts/acceptance/verify_route_acceptance_matrix.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck disable=SC1091
+source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"
+
+CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
+TS="${TS:-$(timestamp_token)}"
+MATRIX_DIR="${MATRIX_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_matrix}"
+
+mkdir -p "$MATRIX_DIR"
+
+run_step() {
+  local name="$1"
+  shift
+  echo "==> $name"
+  ARTIFACT_DIR="$MATRIX_DIR/$name" "$@"
+}
+
+run_step control_plane bash "$ROOT_DIR/scripts/acceptance/verify_route_control_plane.sh"
+run_step health_ui bash "$ROOT_DIR/scripts/acceptance/verify_route_health_ui.sh"
+run_step data_plane bash "$ROOT_DIR/scripts/acceptance/verify_route_data_plane.sh"
+
+python3 - "$MATRIX_DIR" >"$MATRIX_DIR/summary.json" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+matrix_dir = Path(sys.argv[1])
+control = json.loads((matrix_dir / "control_plane" / "10-summary.json").read_text())
+health = json.loads((matrix_dir / "health_ui" / "12-summary.json").read_text())
+data = json.loads((matrix_dir / "data_plane" / "07-summary.json").read_text())
+
+summary = {
+    "matrix_dir": str(matrix_dir),
+    "control_plane_group_id": control["group_id"],
+    "health_ui_group_id": health["group_id"],
+    "data_plane_group_id": data["group_id"],
+    "data_plane_request_id": data["request_id"],
+    "data_plane_upstream_status": data["forward_upstream_status"],
+    "health_ui_resolve_route_id": health["resolve_route_id"],
+}
+print(json.dumps(summary, ensure_ascii=False, indent=2))
+PY
+
+cat "$MATRIX_DIR/summary.json"

scripts/acceptance/verify_route_control_plane.sh

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck disable=SC1091
+source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"
+
+CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
+TS="${TS:-$(timestamp_token)}"
+ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_control_plane}"
+
+GROUP_ID="${GROUP_ID:-p2t4-cp-${TS}}"
+ROUTE_ID="${ROUTE_ID:-primary-${TS}}"
+PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}"
+SHADOW_MODEL="${SHADOW_MODEL:-gpt-5.4}"
+SHADOW_HOST_ID="${SHADOW_HOST_ID:-shadow-host-${TS}}"
+SHADOW_GROUP_ID="${SHADOW_GROUP_ID:-shadow-group-${TS}}"
+
+crm_auth_init
+ensure_artifact_dir
+
+create_group_payload="$(python3 - "$GROUP_ID" <<'PY'
+import json, sys
+group_id = sys.argv[1]
+print(json.dumps({
+    "logical_group_id": group_id,
+    "display_name": f"P2T4 Control Plane {group_id}",
+    "status": "active",
+    "description": "P2-T4 control plane verification group",
+    "route_policy": "priority",
+    "sticky_mode": "conversation_preferred",
+    "conversation_ttl_seconds": 1200,
+    "user_model_ttl_seconds": 600,
+    "failover_threshold": 2,
+    "cooldown_seconds": 300,
+}, ensure_ascii=False))
+PY
+)"
+save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")"
+
+add_model_payload="$(python3 - "$PUBLIC_MODEL" <<'PY'
+import json, sys
+print(json.dumps({"public_model": sys.argv[1], "status": "active"}, ensure_ascii=False))
+PY
+)"
+save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "$add_model_payload")"
+
+create_route_payload="$(python3 - "$ROUTE_ID" "$SHADOW_GROUP_ID" "$SHADOW_HOST_ID" <<'PY'
+import json, sys
+route_id, shadow_group_id, shadow_host_id = sys.argv[1:4]
+print(json.dumps({
+    "route_id": route_id,
+    "name": f"Primary {route_id}",
+    "status": "active",
+    "priority": 10,
+    "weight": 100,
+    "shadow_group_id": shadow_group_id,
+    "shadow_host_id": shadow_host_id,
+    "upstream_base_url_hint": "https://primary.example/v1",
+}, ensure_ascii=False))
+PY
+)"
+save_json 03-create-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "$create_route_payload")"
+
+add_route_model_payload="$(python3 - "$PUBLIC_MODEL" "$SHADOW_MODEL" <<'PY'
+import json, sys
+public_model, shadow_model = sys.argv[1:3]
+print(json.dumps({
+    "public_model": public_model,
+    "shadow_model": shadow_model,
+    "status": "active",
+}, ensure_ascii=False))
+PY
+)"
+save_json 04-add-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID/models" "$add_route_model_payload")"
+
+save_json 05-get-group "$(crm_curl_json GET "/api/logical-groups/$GROUP_ID")"
+
+update_group_payload="$(python3 - "$GROUP_ID" <<'PY'
+import json, sys
+group_id = sys.argv[1]
+print(json.dumps({
+    "display_name": f"P2T4 Control Plane Updated {group_id}",
+    "status": "active",
+    "description": "P2-T4 control plane verification group updated",
+    "route_policy": "priority",
+    "sticky_mode": "conversation_preferred",
+    "conversation_ttl_seconds": 1500,
+    "user_model_ttl_seconds": 900,
+    "failover_threshold": 2,
+    "cooldown_seconds": 360,
+}, ensure_ascii=False))
+PY
+)"
+save_json 06-update-group "$(crm_curl_json PUT "/api/logical-groups/$GROUP_ID" "$update_group_payload")"
+
+update_route_payload="$(python3 - "$SHADOW_GROUP_ID" "$SHADOW_HOST_ID" <<'PY'
+import json, sys
+shadow_group_id, shadow_host_id = sys.argv[1:3]
+print(json.dumps({
+    "name": "Primary Route Updated",
+    "status": "active",
+    "priority": 12,
+    "weight": 80,
+    "shadow_group_id": shadow_group_id,
+    "shadow_host_id": shadow_host_id,
+    "upstream_base_url_hint": "https://primary-updated.example/v1",
+    "cooldown_until": "",
+}, ensure_ascii=False))
+PY
+)"
+save_json 07-update-route "$(crm_curl_json PUT "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID" "$update_route_payload")"
+
+save_json 08-list-routes "$(crm_curl_json GET "/api/logical-groups/$GROUP_ID/routes")"
+save_json 09-list-route-models "$(crm_curl_json GET "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID/models")"
+
+python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$ROUTE_ID" "$PUBLIC_MODEL" "$SHADOW_MODEL" "$SHADOW_HOST_ID" "$SHADOW_GROUP_ID" >"$ARTIFACT_DIR/10-summary.json" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+art_dir, group_id, route_id, public_model, shadow_model, shadow_host_id, shadow_group_id = sys.argv[1:8]
+art = Path(art_dir)
+create_group = json.loads((art / "01-create-group.json").read_text())["logical_group"]
+update_group = json.loads((art / "06-update-group.json").read_text())["logical_group"]
+update_route = json.loads((art / "07-update-route.json").read_text())["logical_group_route"]
+list_routes = json.loads((art / "08-list-routes.json").read_text())["routes"]
+route_models = json.loads((art / "09-list-route-models.json").read_text())["models"]
+
+assert create_group["logical_group_id"] == group_id
+assert update_group["display_name"].startswith("P2T4 Control Plane Updated")
+assert update_route["route_id"] == route_id
+assert update_route["weight"] == 80
+assert update_route["shadow_host_id"] == shadow_host_id
+assert update_route["shadow_group_id"] == shadow_group_id
+assert any(item["route_id"] == route_id for item in list_routes)
+assert any(item["public_model"] == public_model and item["shadow_model"] == shadow_model for item in route_models)
+
+summary = {
+    "group_id": group_id,
+    "route_id": route_id,
+    "public_model": public_model,
+    "shadow_model": shadow_model,
+    "shadow_host_id": shadow_host_id,
+    "shadow_group_id": shadow_group_id,
+    "checks": {
+        "group_created": True,
+        "group_updated": True,
+        "route_created": True,
+        "route_updated": True,
+        "route_model_created": True,
+        "route_model_listed": True,
+    },
+}
+print(json.dumps(summary, ensure_ascii=False, indent=2))
+PY
+
+cat "$ARTIFACT_DIR/10-summary.json"

scripts/acceptance/verify_route_data_plane.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck disable=SC1091
+source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"
+
+CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
+TS="${TS:-$(timestamp_token)}"
+ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_data_plane}"
+
+GROUP_ID="${GROUP_ID:-p2t4-dp-${TS}}"
+ROUTE_ID="${ROUTE_ID:-primary-${TS}}"
+PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}"
+SHADOW_MODEL="${SHADOW_MODEL:-gpt-5.4}"
+SHADOW_HOST_ID="${SHADOW_HOST_ID:?SHADOW_HOST_ID required}"
+SHADOW_GROUP_ID="${SHADOW_GROUP_ID:?SHADOW_GROUP_ID required}"
+REQUEST_ID="${REQUEST_ID:-req-p2t4-dp-${TS}}"
+SUBJECT_ID="${SUBJECT_ID:-conv-p2t4-dp-${TS}}"
+SUBSCRIPTION_USER_ID="${SUBSCRIPTION_USER_ID:-}"
+GATEWAY_API_KEY="${GATEWAY_API_KEY:-}"
+
+if [[ -z "$SUBSCRIPTION_USER_ID" && -z "$GATEWAY_API_KEY" ]]; then
+  echo "missing data-plane auth: set SUBSCRIPTION_USER_ID or GATEWAY_API_KEY" >&2
+  exit 1
+fi
+
+crm_auth_init
+ensure_artifact_dir
+
+create_group_payload="$(python3 - "$GROUP_ID" <<'PY'
+import json, sys
+group_id = sys.argv[1]
+print(json.dumps({
+    "logical_group_id": group_id,
+    "display_name": f"P2T4 Data Plane {group_id}",
+    "status": "active",
+    "description": "P2-T4 data plane verification group",
+    "route_policy": "priority",
+    "sticky_mode": "conversation_preferred",
+    "conversation_ttl_seconds": 1200,
+    "user_model_ttl_seconds": 600,
+    "failover_threshold": 2,
+    "cooldown_seconds": 300,
+}, ensure_ascii=False))
+PY
+)"
+save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")"
+save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")"
+save_json 03-create-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "{\"route_id\":\"$ROUTE_ID\",\"name\":\"Primary $ROUTE_ID\",\"status\":\"active\",\"priority\":10,\"weight\":100,\"shadow_group_id\":\"$SHADOW_GROUP_ID\",\"shadow_host_id\":\"$SHADOW_HOST_ID\",\"upstream_base_url_hint\":\"https://real-shadow.example/v1\"}")"
+save_json 04-add-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"shadow_model\":\"$SHADOW_MODEL\",\"status\":\"active\"}")"
+
+chat_payload="$(python3 - "$GROUP_ID" "$PUBLIC_MODEL" "$REQUEST_ID" "$SUBJECT_ID" "$SUBSCRIPTION_USER_ID" "$GATEWAY_API_KEY" <<'PY'
+import json, sys
+logical_group_id, model, request_id, subject_id, subscription_user_id, gateway_api_key = sys.argv[1:7]
+payload = {
+    "logical_group_id": logical_group_id,
+    "model": model,
+    "request_id": request_id,
+    "scope": "conversation",
+    "subject_id": subject_id,
+    "messages": [{"role": "user", "content": "ping"}],
+    "sync": True,
+}
+if subscription_user_id:
+    payload["subscription_user_id"] = subscription_user_id
+if gateway_api_key:
+    payload["gateway_api_key"] = gateway_api_key
+print(json.dumps(payload, ensure_ascii=False))
+PY
+)"
+save_json 05-route-chat "$(crm_curl_json POST "/api/routing/chat/completions" "$chat_payload")"
+save_json 06-decision-logs "$(crm_curl_json GET "/api/routing/logs/decisions?request_id=$REQUEST_ID&limit=5")"
+
+python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$ROUTE_ID" "$PUBLIC_MODEL" "$SHADOW_MODEL" "$SHADOW_HOST_ID" "$SHADOW_GROUP_ID" "$REQUEST_ID" "$SUBSCRIPTION_USER_ID" "$GATEWAY_API_KEY" >"$ARTIFACT_DIR/07-summary.json" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+(
+    art_dir,
+    group_id,
+    route_id,
+    public_model,
+    shadow_model,
+    shadow_host_id,
+    shadow_group_id,
+    request_id,
+    subscription_user_id,
+    gateway_api_key,
+) = sys.argv[1:11]
+art = Path(art_dir)
+chat = json.loads((art / "05-route-chat.json").read_text())
+logs = json.loads((art / "06-decision-logs.json").read_text())["decision_logs"]
+
+assert chat["request_id"] == request_id
+assert chat["logical_group_id"] == group_id
+assert chat["model"] == public_model
+assert chat["selected_route"]["route_id"] == route_id
+assert chat["selected_route"]["shadow_host_id"] == shadow_host_id
+assert chat["selected_route"]["shadow_group_id"] == shadow_group_id
+assert chat["selected_route"]["shadow_model"] == shadow_model
+assert chat["forward"]["upstream_status"] == 200
+assert logs, logs
+assert logs[0]["request_id"] == request_id
+assert logs[0]["selected_route_id"] == route_id
+
+expected_source = "managed_subscription" if subscription_user_id else "provided_gateway_key"
+summary = {
+    "group_id": group_id,
+    "route_id": route_id,
+    "request_id": request_id,
+    "auth_mode": expected_source,
+    "forward_upstream_status": chat["forward"]["upstream_status"],
+    "selected_shadow_host_id": chat["selected_route"]["shadow_host_id"],
+    "selected_shadow_group_id": chat["selected_route"]["shadow_group_id"],
+    "selected_shadow_model": chat["selected_route"]["shadow_model"],
+    "effective_gateway_key_source": chat["forward"].get("effective_gateway_key_source"),
+    "decision_log_count": len(logs),
+}
+print(json.dumps(summary, ensure_ascii=False, indent=2))
+PY
+
+cat "$ARTIFACT_DIR/07-summary.json"

scripts/acceptance/verify_route_health_ui.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck disable=SC1091
+source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"
+
+CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
+ROUTE_HEALTH_PAGE_URL="${ROUTE_HEALTH_PAGE_URL:-https://sub.tksea.top/portal/admin/route-health.html}"
+TS="${TS:-$(timestamp_token)}"
+ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_health_ui}"
+
+GROUP_ID="${GROUP_ID:-p2t4-health-${TS}}"
+PRIMARY_ROUTE="${PRIMARY_ROUTE:-primary-${TS}}"
+FALLBACK_ROUTE="${FALLBACK_ROUTE:-fallback-${TS}}"
+FAILING_ROUTE="${FAILING_ROUTE:-failing-${TS}}"
+PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}"
+REQUEST_ID="${REQUEST_ID:-req-p2t4-health-${TS}}"
+SUBJECT_ID="${SUBJECT_ID:-conv-p2t4-health-${TS}}"
+
+crm_auth_init
+ensure_artifact_dir
+
+curl_status_to_file "$ROUTE_HEALTH_PAGE_URL" "$ARTIFACT_DIR/00-route-health.html"
+
+create_group_payload="$(python3 - "$GROUP_ID" <<'PY'
+import json, sys
+group_id = sys.argv[1]
+print(json.dumps({
+    "logical_group_id": group_id,
+    "display_name": f"P2T4 Health {group_id}",
+    "status": "active",
+    "description": "P2-T4 health verification group",
+    "route_policy": "priority",
+    "sticky_mode": "conversation_preferred",
+    "conversation_ttl_seconds": 1200,
+    "user_model_ttl_seconds": 600,
+    "failover_threshold": 2,
+    "cooldown_seconds": 300,
+}, ensure_ascii=False))
+PY
+)"
+save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")"
+save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")"
+
+create_route() {
+  local route_id="$1"
+  local route_name="$2"
+  local priority="$3"
+  local shadow_group_id="$4"
+  local shadow_host_id="$5"
+  crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" \
+    "{\"route_id\":\"$route_id\",\"name\":\"$route_name\",\"status\":\"active\",\"priority\":$priority,\"weight\":100,\"shadow_group_id\":\"$shadow_group_id\",\"shadow_host_id\":\"$shadow_host_id\",\"upstream_base_url_hint\":\"https://$route_id.example/v1\"}"
+}
+
+save_json 03-create-primary-route "$(create_route "$PRIMARY_ROUTE" "Primary Route" 10 "shadow-primary-$TS" "shadow-host-primary-$TS")"
+save_json 04-create-fallback-route "$(create_route "$FALLBACK_ROUTE" "Fallback Route" 20 "shadow-fallback-$TS" "shadow-host-fallback-$TS")"
+save_json 05-create-failing-route "$(create_route "$FAILING_ROUTE" "Failing Route" 30 "shadow-failing-$TS" "shadow-host-failing-$TS")"
+
+for route_id in "$PRIMARY_ROUTE" "$FALLBACK_ROUTE" "$FAILING_ROUTE"; do
+  save_json "route-model-${route_id}" "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$route_id/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"shadow_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")"
+done
+
+save_json 06-set-cooldown "$(crm_curl_json POST "/api/routing/sticky/cooldowns" "{\"route_id\":\"$PRIMARY_ROUTE\",\"reason\":\"degraded\",\"ttl_seconds\":600}")"
+save_json 07-set-failure "$(crm_curl_json POST "/api/routing/sticky/route-failures" "{\"route_id\":\"$FAILING_ROUTE\",\"failure_count\":2,\"last_error_class\":\"timeout\",\"ttl_seconds\":600}")"
+save_json 08-health-before "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")"
+save_json 09-resolve "$(crm_curl_json POST "/api/routing/resolve" "{\"request_id\":\"$REQUEST_ID\",\"logical_group_id\":\"$GROUP_ID\",\"public_model\":\"$PUBLIC_MODEL\",\"scope\":\"conversation\",\"subject_id\":\"$SUBJECT_ID\",\"sync\":true}")"
+save_json 10-health-after "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")"
+save_json 11-failovers "$(crm_curl_json GET "/api/routing/logs/failovers?request_id=$REQUEST_ID&limit=5")"
+
+python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$PRIMARY_ROUTE" "$FALLBACK_ROUTE" "$FAILING_ROUTE" "$REQUEST_ID" >"$ARTIFACT_DIR/12-summary.json" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+art_dir, group_id, primary_route, fallback_route, failing_route, request_id = sys.argv[1:7]
+art = Path(art_dir)
+page = (art / "00-route-health.html").read_text()
+before = json.loads((art / "08-health-before.json").read_text())["route_health"]
+resolve = json.loads((art / "09-resolve.json").read_text())["resolve"]
+after = json.loads((art / "10-health-after.json").read_text())["route_health"]
+failovers = json.loads((art / "11-failovers.json").read_text())["failover_events"]
+
+def by_id(items):
+    return {item["route_id"]: item for item in items}
+
+before_map = by_id(before)
+after_map = by_id(after)
+
+assert "Route Health Admin" in page
+assert before_map[primary_route]["runtime_status"] == "cooldown"
+assert before_map[failing_route]["runtime_status"] == "failing"
+assert resolve["route_id"] == fallback_route
+assert resolve["fallback_used"] is True
+assert after_map[primary_route]["runtime_status"] == "cooldown"
+assert after_map[fallback_route]["runtime_status"] == "healthy"
+assert after_map[fallback_route]["recent_failover_count"] >= 1
+assert failovers and failovers[0]["from_route_id"] == primary_route
+assert failovers[0]["to_route_id"] == fallback_route
+assert failovers[0]["reason"] == "active_cooldown:degraded"
+
+summary = {
+    "group_id": group_id,
+    "request_id": request_id,
+    "resolve_route_id": resolve["route_id"],
+    "resolve_fallback_used": resolve["fallback_used"],
+    "before_statuses": {k: v["runtime_status"] for k, v in before_map.items()},
+    "after_statuses": {k: v["runtime_status"] for k, v in after_map.items()},
+    "fallback_recent_failover_count": after_map[fallback_route]["recent_failover_count"],
+}
+print(json.dumps(summary, ensure_ascii=False, indent=2))
+PY
+
+cat "$ARTIFACT_DIR/12-summary.json"

scripts/test/test_real_host_scripts.sh

@@ -660,6 +660,258 @@ EOF
   [[ -f "$sensitive_root/20260522_foo/05-subscription-access-prep.sql" ]] || fail "sql file was not moved to sensitive mirror"
 }
 
+run_test_verify_route_control_plane_script() {
+  local tmpdir fakebin artifact_dir stdout_file
+  tmpdir="$(mktemp -d)"
+  trap 'rm -rf "$tmpdir"' RETURN
+  fakebin="$tmpdir/bin"
+  artifact_dir="$tmpdir/artifacts"
+  stdout_file="$tmpdir/verify_route_control_plane.stdout.txt"
+  mkdir -p "$fakebin" "$artifact_dir"
+
+  cat > "$fakebin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+method="GET"
+url=""
+payload=""
+prev=""
+for arg in "$@"; do
+  case "$prev" in
+    -X) method="$arg"; prev=""; continue ;;
+    -d|--data) payload="$arg"; prev=""; continue ;;
+  esac
+  case "$arg" in
+    -X|-d|--data) prev="$arg"; continue ;;
+    http://*|https://*) url="$arg" ;;
+  esac
+done
+case "$method $url" in
+  "POST http://crm.example.com/api/logical-groups")
+    printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane p2t4-cp-1700000000","status":"active"}}'
+    ;;
+  "POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/models")
+    printf '%s\n' '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}'
+    ;;
+  "POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes")
+    printf '%s\n' '{"logical_group_route":{"route_id":"primary-1700000000","logical_group_id":"p2t4-cp-1700000000","name":"Primary primary-1700000000","status":"active","priority":10,"weight":100,"shadow_group_id":"shadow-group-1700000000","shadow_host_id":"shadow-host-1700000000"}}'
+    ;;
+  "POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000/models")
+    printf '%s\n' '{"logical_group_route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4","status":"active"}}'
+    ;;
+  "GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000")
+    printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane p2t4-cp-1700000000","status":"active","models":[{"public_model":"gpt-5.4"}],"routes":[{"route_id":"primary-1700000000"}]}}'
+    ;;
+  "PUT http://crm.example.com/api/logical-groups/p2t4-cp-1700000000")
+    printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane Updated p2t4-cp-1700000000","status":"active"}}'
+    ;;
+  "PUT http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000")
+    printf '%s\n' '{"logical_group_route":{"route_id":"primary-1700000000","logical_group_id":"p2t4-cp-1700000000","name":"Primary Route Updated","status":"active","priority":12,"weight":80,"shadow_group_id":"shadow-group-1700000000","shadow_host_id":"shadow-host-1700000000"}}'
+    ;;
+  "GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes")
+    printf '%s\n' '{"routes":[{"route_id":"primary-1700000000","weight":80}]}'
+    ;;
+  "GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000/models")
+    printf '%s\n' '{"models":[{"public_model":"gpt-5.4","shadow_model":"gpt-5.4","status":"active"}]}'
+    ;;
+  *)
+    echo "unexpected curl request: $method $url payload=$payload" >&2
+    exit 1
+    ;;
+esac
+EOF
+  chmod +x "$fakebin/curl"
+
+  PATH="$fakebin:$PATH" \
+  CRM_BASE="http://crm.example.com" \
+  CRM_ADMIN_TOKEN="token" \
+  TS="1700000000" \
+  ARTIFACT_DIR="$artifact_dir" \
+  bash "$ROOT_DIR/scripts/acceptance/verify_route_control_plane.sh" >"$stdout_file"
+
+  local summary stdout_text
+  summary="$(cat "$artifact_dir/10-summary.json")"
+  stdout_text="$(cat "$stdout_file")"
+  assert_contains "$summary" '"group_id": "p2t4-cp-1700000000"'
+  assert_contains "$summary" '"route_id": "primary-1700000000"'
+  assert_contains "$summary" '"route_updated": true'
+  assert_contains "$stdout_text" '"route_model_listed": true'
+}
+
+run_test_verify_route_data_plane_script() {
+  local tmpdir fakebin artifact_dir stdout_file payload_log
+  tmpdir="$(mktemp -d)"
+  trap 'rm -rf "$tmpdir"' RETURN
+  fakebin="$tmpdir/bin"
+  artifact_dir="$tmpdir/artifacts"
+  stdout_file="$tmpdir/verify_route_data_plane.stdout.txt"
+  payload_log="$tmpdir/payload.log"
+  mkdir -p "$fakebin" "$artifact_dir"
+
+  cat > "$fakebin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+method="GET"
+url=""
+payload=""
+prev=""
+for arg in "$@"; do
+  case "$prev" in
+    -X) method="$arg"; prev=""; continue ;;
+    -d|--data) payload="$arg"; prev=""; continue ;;
+  esac
+  case "$arg" in
+    -X|-d|--data) prev="$arg"; continue ;;
+    http://*|https://*) url="$arg" ;;
+  esac
+done
+printf '%s\n' "$payload" >> "${PAYLOAD_LOG:?missing PAYLOAD_LOG}"
+case "$method $url" in
+  "POST http://crm.example.com/api/logical-groups")
+    printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-dp-1700000001","display_name":"P2T4 Data Plane p2t4-dp-1700000001","status":"active"}}'
+    ;;
+  "POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/models")
+    printf '%s\n' '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}'
+    ;;
+  "POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/routes")
+    printf '%s\n' '{"logical_group_route":{"route_id":"primary-1700000001","shadow_group_id":"shadow-group-9","shadow_host_id":"shadow-host-real"}}'
+    ;;
+  "POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/routes/primary-1700000001/models")
+    printf '%s\n' '{"logical_group_route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4"}}'
+    ;;
+  "POST http://crm.example.com/api/routing/chat/completions")
+    printf '%s\n' '{"request_id":"req-p2t4-dp-1700000001","logical_group_id":"p2t4-dp-1700000001","model":"gpt-5.4","selected_route":{"route_id":"primary-1700000001","shadow_host_id":"shadow-host-real","shadow_group_id":"shadow-group-9","shadow_model":"gpt-5.4"},"forward":{"upstream_status":200,"effective_gateway_key_source":"managed_subscription"}}'
+    ;;
+  "GET http://crm.example.com/api/routing/logs/decisions?request_id=req-p2t4-dp-1700000001&limit=5")
+    printf '%s\n' '{"decision_logs":[{"request_id":"req-p2t4-dp-1700000001","selected_route_id":"primary-1700000001"}]}'
+    ;;
+  *)
+    echo "unexpected curl request: $method $url payload=$payload" >&2
+    exit 1
+    ;;
+esac
+EOF
+  chmod +x "$fakebin/curl"
+
+  PATH="$fakebin:$PATH" \
+  PAYLOAD_LOG="$payload_log" \
+  CRM_BASE="http://crm.example.com" \
+  CRM_ADMIN_TOKEN="token" \
+  TS="1700000001" \
+  SHADOW_HOST_ID="shadow-host-real" \
+  SHADOW_GROUP_ID="shadow-group-9" \
+  SUBSCRIPTION_USER_ID="36" \
+  ARTIFACT_DIR="$artifact_dir" \
+  bash "$ROOT_DIR/scripts/acceptance/verify_route_data_plane.sh" >"$stdout_file"
+
+  local summary payloads
+  summary="$(cat "$artifact_dir/07-summary.json")"
+  payloads="$(cat "$payload_log")"
+  assert_contains "$summary" '"forward_upstream_status": 200'
+  assert_contains "$summary" '"effective_gateway_key_source": "managed_subscription"'
+  assert_contains "$payloads" '"subscription_user_id": "36"'
+}
+
+run_test_verify_route_health_ui_script() {
+  local tmpdir fakebin artifact_dir stdout_file
+  tmpdir="$(mktemp -d)"
+  trap 'rm -rf "$tmpdir"' RETURN
+  fakebin="$tmpdir/bin"
+  artifact_dir="$tmpdir/artifacts"
+  stdout_file="$tmpdir/verify_route_health_ui.stdout.txt"
+  mkdir -p "$fakebin" "$artifact_dir"
+
+  cat > "$fakebin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+method="GET"
+url=""
+payload=""
+output_file=""
+prev=""
+for arg in "$@"; do
+  case "$prev" in
+    -X) method="$arg"; prev=""; continue ;;
+    -d|--data) payload="$arg"; prev=""; continue ;;
+    -o) output_file="$arg"; prev=""; continue ;;
+  esac
+  case "$arg" in
+    -X|-d|--data|-o) prev="$arg"; continue ;;
+    http://*|https://*) url="$arg" ;;
+  esac
+done
+write_body() {
+  local body="$1"
+  if [[ -n "$output_file" ]]; then
+    printf '%s\n' "$body" > "$output_file"
+  else
+    printf '%s\n' "$body"
+  fi
+}
+case "$method $url" in
+  "GET http://portal.example.com/route-health.html")
+    write_body '<html><title>Route Health Admin</title><body>Route Health Admin</body></html>'
+    ;;
+  "POST http://crm.example.com/api/logical-groups")
+    write_body '{"logical_group":{"logical_group_id":"p2t4-health-1700000002","status":"active"}}'
+    ;;
+  "POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/models")
+    write_body '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}'
+    ;;
+  "POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes")
+    if [[ "$payload" == *'"route_id":"primary-1700000002"'* ]]; then
+      write_body '{"logical_group_route":{"route_id":"primary-1700000002"}}'
+    elif [[ "$payload" == *'"route_id":"fallback-1700000002"'* ]]; then
+      write_body '{"logical_group_route":{"route_id":"fallback-1700000002"}}'
+    else
+      write_body '{"logical_group_route":{"route_id":"failing-1700000002"}}'
+    fi
+    ;;
+  "POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/primary-1700000002/models"|"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/fallback-1700000002/models"|"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/failing-1700000002/models")
+    write_body '{"logical_group_route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4"}}'
+    ;;
+  "POST http://crm.example.com/api/routing/sticky/cooldowns")
+    write_body '{"route_cooldown":{"route_id":"primary-1700000002","reason":"degraded"}}'
+    ;;
+  "POST http://crm.example.com/api/routing/sticky/route-failures")
+    write_body '{"route_failure":{"route_id":"failing-1700000002","failure_count":2,"last_error_class":"timeout"}}'
+    ;;
+  "GET http://crm.example.com/api/routing/routes/health?logical_group_id=p2t4-health-1700000002")
+    if [[ ! -f /tmp/p2t4-health-switch ]]; then
+      write_body '{"route_health":[{"route_id":"primary-1700000002","runtime_status":"cooldown"},{"route_id":"fallback-1700000002","runtime_status":"healthy","recent_failover_count":0},{"route_id":"failing-1700000002","runtime_status":"failing"}]}'
+    else
+      write_body '{"route_health":[{"route_id":"primary-1700000002","runtime_status":"cooldown"},{"route_id":"fallback-1700000002","runtime_status":"healthy","recent_failover_count":1,"last_selected_at":"2026-05-29T12:00:00Z"},{"route_id":"failing-1700000002","runtime_status":"failing"}]}'
+    fi
+    ;;
+  "POST http://crm.example.com/api/routing/resolve")
+    : > /tmp/p2t4-health-switch
+    write_body '{"resolve":{"request_id":"req-p2t4-health-1700000002","route_id":"fallback-1700000002","fallback_used":true}}'
+    ;;
+  "GET http://crm.example.com/api/routing/logs/failovers?request_id=req-p2t4-health-1700000002&limit=5")
+    write_body '{"failover_events":[{"from_route_id":"primary-1700000002","to_route_id":"fallback-1700000002","reason":"active_cooldown:degraded"}]}'
+    ;;
+  *)
+    echo "unexpected curl request: $method $url payload=$payload" >&2
+    exit 1
+    ;;
+esac
+EOF
+  chmod +x "$fakebin/curl"
+
+  PATH="$fakebin:$PATH" \
+  CRM_BASE="http://crm.example.com" \
+  CRM_ADMIN_TOKEN="token" \
+  ROUTE_HEALTH_PAGE_URL="http://portal.example.com/route-health.html" \
+  TS="1700000002" \
+  ARTIFACT_DIR="$artifact_dir" \
+  bash "$ROOT_DIR/scripts/acceptance/verify_route_health_ui.sh" >"$stdout_file"
+
+  local summary
+  summary="$(cat "$artifact_dir/12-summary.json")"
+  assert_contains "$summary" '"resolve_route_id": "fallback-1700000002"'
+  assert_contains "$summary" '"fallback_recent_failover_count": 1'
+}
+
 run_test_remote43_patched_stack_renderers() {
   # shellcheck disable=SC1091
   source "$ROOT_DIR/scripts/deploy/remote43_patched_stack_lib.sh"
@@ -784,6 +1036,9 @@ run_test_real_host_acceptance_after_import_hook
 run_test_check_deepseek_completion_split
 run_test_import_remote43_provider_subscription_prep
 run_test_migrate_historical_artifacts
+run_test_verify_route_control_plane_script
+run_test_verify_route_data_plane_script
+run_test_verify_route_health_ui_script
 run_test_remote43_patched_stack_renderers
 run_test_setup_remote43_patched_stack_dry_run