From 949134005ebb10b5f176caa9d5e82809e7cded62 Mon Sep 17 00:00:00 2001 From: phamnazage-jpg Date: Fri, 29 May 2026 13:50:16 +0800 Subject: [PATCH] feat(routing): add route acceptance matrix scripts --- docs/ROUTE_ACCEPTANCE_MATRIX.md | 165 ++++++++++++ scripts/acceptance/route_acceptance_lib.sh | 131 +++++++++ .../verify_route_acceptance_matrix.sh | 47 ++++ .../acceptance/verify_route_control_plane.sh | 158 +++++++++++ scripts/acceptance/verify_route_data_plane.sh | 124 +++++++++ scripts/acceptance/verify_route_health_ui.sh | 114 ++++++++ scripts/test/test_real_host_scripts.sh | 255 ++++++++++++++++++ 7 files changed, 994 insertions(+) create mode 100644 docs/ROUTE_ACCEPTANCE_MATRIX.md create mode 100644 scripts/acceptance/route_acceptance_lib.sh create mode 100755 scripts/acceptance/verify_route_acceptance_matrix.sh create mode 100755 scripts/acceptance/verify_route_control_plane.sh create mode 100755 scripts/acceptance/verify_route_data_plane.sh create mode 100755 scripts/acceptance/verify_route_health_ui.sh diff --git a/docs/ROUTE_ACCEPTANCE_MATRIX.md b/docs/ROUTE_ACCEPTANCE_MATRIX.md new file mode 100644 index 00000000..498bcd88 --- /dev/null +++ b/docs/ROUTE_ACCEPTANCE_MATRIX.md @@ -0,0 +1,165 @@ +# Route Acceptance Matrix + +日期：2026-05-29 + +## 目标 + +把 Phase 2 已经验证过的三类能力，收敛成固定入口和固定证据格式： + +1. 控制面 +2. 运行态 / 健康视图 +3. 数据面 + +当前统一入口： + +- `scripts/acceptance/verify_route_control_plane.sh` +- `scripts/acceptance/verify_route_health_ui.sh` +- `scripts/acceptance/verify_route_data_plane.sh` +- `scripts/acceptance/verify_route_acceptance_matrix.sh` + +其中 `verify_route_acceptance_matrix.sh` 是总入口，会按顺序执行前三个脚本并汇总 `summary.json`。 + +## 验收矩阵 + +### 1. 控制面 + +脚本： + +- `scripts/acceptance/verify_route_control_plane.sh` + +验证范围： + +- 创建 `logical_group` +- 创建 `logical_group_model` +- 创建 `route` +- 创建 `route_model` +- `GET /api/logical-groups/{group_id}` +- `PUT /api/logical-groups/{group_id}` +- `PUT /api/logical-groups/{group_id}/routes/{route_id}` +- `GET /api/logical-groups/{group_id}/routes` +- `GET /api/logical-groups/{group_id}/routes/{route_id}/models` + +最小输入： + +- `CRM_BASE` +- `CRM_ADMIN_TOKEN` + +主要产物： + +- `01-create-group.json` +- `03-create-route.json` +- `06-update-group.json` +- `07-update-route.json` +- `10-summary.json` + +### 2. 健康视图 / 运行态 + +脚本： + +- `scripts/acceptance/verify_route_health_ui.sh` + +验证范围： + +- 公网 `route-health.html` 页面可达 +- `POST /api/routing/sticky/cooldowns` +- `POST /api/routing/sticky/route-failures` +- `GET /api/routing/routes/health` +- `POST /api/routing/resolve` +- `GET /api/routing/logs/failovers` + +最小输入： + +- `CRM_BASE` +- `CRM_ADMIN_TOKEN` +- 可选 `ROUTE_HEALTH_PAGE_URL` + +主要产物： + +- `00-route-health.html` +- `08-health-before.json` +- `09-resolve.json` +- `10-health-after.json` +- `11-failovers.json` +- `12-summary.json` + +### 3. 数据面 + +脚本： + +- `scripts/acceptance/verify_route_data_plane.sh` + +验证范围： + +- 创建临时 `logical_group / route / route_model` +- `POST /api/routing/chat/completions` +- `GET /api/routing/logs/decisions` + +最小输入： + +- `CRM_BASE` +- `CRM_ADMIN_TOKEN` +- `SHADOW_HOST_ID` +- `SHADOW_GROUP_ID` +- 二选一： + - `SUBSCRIPTION_USER_ID` + - `GATEWAY_API_KEY` + +补充说明： + +- 若传 `SUBSCRIPTION_USER_ID`，期望 `effective_gateway_key_source=managed_subscription` +- 若传 `GATEWAY_API_KEY`，脚本仍可跑通，但这不覆盖“自动供给 managed key”场景 + +主要产物： + +- `05-route-chat.json` +- `06-decision-logs.json` +- `07-summary.json` + +## 总入口 + +脚本： + +- `scripts/acceptance/verify_route_acceptance_matrix.sh` + +它会创建： + +```text +artifacts/phase2-routing-matrix/_route_matrix/ + control_plane/ + health_ui/ + data_plane/ + summary.json +``` + +`summary.json` 只保留最核心的摘要字段： + +- `control_plane_group_id` +- `health_ui_group_id` +- `data_plane_group_id` +- `data_plane_request_id` +- `data_plane_upstream_status` +- `health_ui_resolve_route_id` + +## remote43 推荐执行方式 + +如果要避免把真实 `CRM_ADMIN_TOKEN` 带回本地，推荐直接在 `remote43` 上执行： + +```bash +cd /home/ubuntu/sub2api-kimi-patched-auto2-20260525_18169 +set -a +. ./.env.crm +set +a + +cd /home/ubuntu/sub2api-cn-relay-manager-git-current +CRM_BASE="https://sub.tksea.top/portal-admin-api" \ +SHADOW_HOST_ID="" \ +SHADOW_GROUP_ID="" \ +SUBSCRIPTION_USER_ID="" \ +bash ./scripts/acceptance/verify_route_acceptance_matrix.sh +``` + +## 当前边界 + +- 脚本只负责留证据和判断通过/失败，不负责清理临时 `logical_group` +- 数据面脚本当前仍依赖调用方提供真实 `shadow_host_id / shadow_group_id` +- 尚未覆盖“同公开模型双线路主备”的未来策略矩阵 diff --git a/scripts/acceptance/route_acceptance_lib.sh b/scripts/acceptance/route_acceptance_lib.sh new file mode 100644 index 00000000..e1c9754c --- /dev/null +++ b/scripts/acceptance/route_acceptance_lib.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +ROUTE_MATRIX_ROOT="${ROUTE_MATRIX_ROOT:-$ROOT_DIR/artifacts/phase2-routing-matrix}" + +require_var() { + local name="$1" + if [[ -z "${!name:-}" ]]; then + echo "missing required env: $name" >&2 + exit 1 + fi +} + +ensure_artifact_dir() { + mkdir -p "$ARTIFACT_DIR" +} + +save_json() { + local name="$1" + local payload="$2" + ensure_artifact_dir + printf '%s\n' "$payload" > "$ARTIFACT_DIR/$name.json" +} + +save_text() { + local name="$1" + local payload="$2" + ensure_artifact_dir + printf '%s\n' "$payload" > "$ARTIFACT_DIR/$name" +} + +json_get_file() { + local file="$1" + local key="$2" + python3 - "$file" "$key" <<'PY' +import json +import sys + +file_path, key = sys.argv[1:3] +with open(file_path, 'r', encoding='utf-8') as fh: + value = json.load(fh) +for part in key.split('.'): + if isinstance(value, dict): + value = value.get(part) + else: + value = None + break +if value is None: + raise SystemExit(2) +if isinstance(value, (dict, list)): + print(json.dumps(value, ensure_ascii=False)) +else: + print(value) +PY +} + +crm_auth_init() { + require_var CRM_BASE + + if [[ -n "${CRM_ADMIN_TOKEN:-}" ]]; then + crm_token="$CRM_ADMIN_TOKEN" + crm_cookie_jar="" + return 0 + fi + + if [[ -n "${CRM_COOKIE_JAR:-}" ]]; then + crm_cookie_jar="$CRM_COOKIE_JAR" + else + crm_cookie_jar="$(mktemp /tmp/route-matrix-cookie.XXXXXX.jar)" + fi + rm -f "$crm_cookie_jar" + + require_var CRM_ADMIN_USERNAME + require_var CRM_ADMIN_PASSWORD + + local login_payload + login_payload="$(python3 - "$CRM_ADMIN_USERNAME" "$CRM_ADMIN_PASSWORD" <<'PY' +import json +import sys +username, password = sys.argv[1:3] +print(json.dumps({"username": username, "password": password}, ensure_ascii=False)) +PY +)" + + curl -fsS \ + -c "$crm_cookie_jar" \ + -b "$crm_cookie_jar" \ + -H 'Content-Type: application/json' \ + -X POST \ + "${CRM_BASE%/}/api/admin/session/login" \ + -d "$login_payload" > /dev/null + crm_token="" +} + +crm_curl_json() { + local method="$1" + local path="$2" + local payload="${3:-}" + local -a curl_args + curl_args=(-fsS -X "$method") + if [[ -n "${crm_token:-}" ]]; then + curl_args+=(-H "Authorization: Bearer $crm_token") + elif [[ -n "${crm_cookie_jar:-}" ]]; then + curl_args+=(-b "$crm_cookie_jar" -c "$crm_cookie_jar") + else + echo "missing CRM auth: set CRM_ADMIN_TOKEN or CRM_ADMIN_USERNAME/CRM_ADMIN_PASSWORD" >&2 + exit 2 + fi + if [[ -n "$payload" ]]; then + curl_args+=( + -H 'Content-Type: application/json' + "${CRM_BASE%/}${path}" + -d "$payload" + ) + else + curl_args+=("${CRM_BASE%/}${path}") + fi + curl "${curl_args[@]}" +} + +curl_status_to_file() { + local url="$1" + local file="$2" + ensure_artifact_dir + curl -fsS "$url" -o "$file" +} + +timestamp_token() { + date +%s +} diff --git a/scripts/acceptance/verify_route_acceptance_matrix.sh b/scripts/acceptance/verify_route_acceptance_matrix.sh new file mode 100755 index 00000000..5aa6d7e7 --- /dev/null +++ b/scripts/acceptance/verify_route_acceptance_matrix.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck disable=SC1091 +source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh" + +CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}" +TS="${TS:-$(timestamp_token)}" +MATRIX_DIR="${MATRIX_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_matrix}" + +mkdir -p "$MATRIX_DIR" + +run_step() { + local name="$1" + shift + echo "==> $name" + ARTIFACT_DIR="$MATRIX_DIR/$name" "$@" +} + +run_step control_plane bash "$ROOT_DIR/scripts/acceptance/verify_route_control_plane.sh" +run_step health_ui bash "$ROOT_DIR/scripts/acceptance/verify_route_health_ui.sh" +run_step data_plane bash "$ROOT_DIR/scripts/acceptance/verify_route_data_plane.sh" + +python3 - "$MATRIX_DIR" >"$MATRIX_DIR/summary.json" <<'PY' +import json +import sys +from pathlib import Path + +matrix_dir = Path(sys.argv[1]) +control = json.loads((matrix_dir / "control_plane" / "10-summary.json").read_text()) +health = json.loads((matrix_dir / "health_ui" / "12-summary.json").read_text()) +data = json.loads((matrix_dir / "data_plane" / "07-summary.json").read_text()) + +summary = { + "matrix_dir": str(matrix_dir), + "control_plane_group_id": control["group_id"], + "health_ui_group_id": health["group_id"], + "data_plane_group_id": data["group_id"], + "data_plane_request_id": data["request_id"], + "data_plane_upstream_status": data["forward_upstream_status"], + "health_ui_resolve_route_id": health["resolve_route_id"], +} +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY + +cat "$MATRIX_DIR/summary.json" diff --git a/scripts/acceptance/verify_route_control_plane.sh b/scripts/acceptance/verify_route_control_plane.sh new file mode 100755 index 00000000..25decc00 --- /dev/null +++ b/scripts/acceptance/verify_route_control_plane.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck disable=SC1091 +source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh" + +CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}" +TS="${TS:-$(timestamp_token)}" +ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_control_plane}" + +GROUP_ID="${GROUP_ID:-p2t4-cp-${TS}}" +ROUTE_ID="${ROUTE_ID:-primary-${TS}}" +PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}" +SHADOW_MODEL="${SHADOW_MODEL:-gpt-5.4}" +SHADOW_HOST_ID="${SHADOW_HOST_ID:-shadow-host-${TS}}" +SHADOW_GROUP_ID="${SHADOW_GROUP_ID:-shadow-group-${TS}}" + +crm_auth_init +ensure_artifact_dir + +create_group_payload="$(python3 - "$GROUP_ID" <<'PY' +import json, sys +group_id = sys.argv[1] +print(json.dumps({ + "logical_group_id": group_id, + "display_name": f"P2T4 Control Plane {group_id}", + "status": "active", + "description": "P2-T4 control plane verification group", + "route_policy": "priority", + "sticky_mode": "conversation_preferred", + "conversation_ttl_seconds": 1200, + "user_model_ttl_seconds": 600, + "failover_threshold": 2, + "cooldown_seconds": 300, +}, ensure_ascii=False)) +PY +)" +save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")" + +add_model_payload="$(python3 - "$PUBLIC_MODEL" <<'PY' +import json, sys +print(json.dumps({"public_model": sys.argv[1], "status": "active"}, ensure_ascii=False)) +PY +)" +save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "$add_model_payload")" + +create_route_payload="$(python3 - "$ROUTE_ID" "$SHADOW_GROUP_ID" "$SHADOW_HOST_ID" <<'PY' +import json, sys +route_id, shadow_group_id, shadow_host_id = sys.argv[1:4] +print(json.dumps({ + "route_id": route_id, + "name": f"Primary {route_id}", + "status": "active", + "priority": 10, + "weight": 100, + "shadow_group_id": shadow_group_id, + "shadow_host_id": shadow_host_id, + "upstream_base_url_hint": "https://primary.example/v1", +}, ensure_ascii=False)) +PY +)" +save_json 03-create-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "$create_route_payload")" + +add_route_model_payload="$(python3 - "$PUBLIC_MODEL" "$SHADOW_MODEL" <<'PY' +import json, sys +public_model, shadow_model = sys.argv[1:3] +print(json.dumps({ + "public_model": public_model, + "shadow_model": shadow_model, + "status": "active", +}, ensure_ascii=False)) +PY +)" +save_json 04-add-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID/models" "$add_route_model_payload")" + +save_json 05-get-group "$(crm_curl_json GET "/api/logical-groups/$GROUP_ID")" + +update_group_payload="$(python3 - "$GROUP_ID" <<'PY' +import json, sys +group_id = sys.argv[1] +print(json.dumps({ + "display_name": f"P2T4 Control Plane Updated {group_id}", + "status": "active", + "description": "P2-T4 control plane verification group updated", + "route_policy": "priority", + "sticky_mode": "conversation_preferred", + "conversation_ttl_seconds": 1500, + "user_model_ttl_seconds": 900, + "failover_threshold": 2, + "cooldown_seconds": 360, +}, ensure_ascii=False)) +PY +)" +save_json 06-update-group "$(crm_curl_json PUT "/api/logical-groups/$GROUP_ID" "$update_group_payload")" + +update_route_payload="$(python3 - "$SHADOW_GROUP_ID" "$SHADOW_HOST_ID" <<'PY' +import json, sys +shadow_group_id, shadow_host_id = sys.argv[1:3] +print(json.dumps({ + "name": "Primary Route Updated", + "status": "active", + "priority": 12, + "weight": 80, + "shadow_group_id": shadow_group_id, + "shadow_host_id": shadow_host_id, + "upstream_base_url_hint": "https://primary-updated.example/v1", + "cooldown_until": "", +}, ensure_ascii=False)) +PY +)" +save_json 07-update-route "$(crm_curl_json PUT "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID" "$update_route_payload")" + +save_json 08-list-routes "$(crm_curl_json GET "/api/logical-groups/$GROUP_ID/routes")" +save_json 09-list-route-models "$(crm_curl_json GET "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID/models")" + +python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$ROUTE_ID" "$PUBLIC_MODEL" "$SHADOW_MODEL" "$SHADOW_HOST_ID" "$SHADOW_GROUP_ID" >"$ARTIFACT_DIR/10-summary.json" <<'PY' +import json +import sys +from pathlib import Path + +art_dir, group_id, route_id, public_model, shadow_model, shadow_host_id, shadow_group_id = sys.argv[1:8] +art = Path(art_dir) +create_group = json.loads((art / "01-create-group.json").read_text())["logical_group"] +update_group = json.loads((art / "06-update-group.json").read_text())["logical_group"] +update_route = json.loads((art / "07-update-route.json").read_text())["logical_group_route"] +list_routes = json.loads((art / "08-list-routes.json").read_text())["routes"] +route_models = json.loads((art / "09-list-route-models.json").read_text())["models"] + +assert create_group["logical_group_id"] == group_id +assert update_group["display_name"].startswith("P2T4 Control Plane Updated") +assert update_route["route_id"] == route_id +assert update_route["weight"] == 80 +assert update_route["shadow_host_id"] == shadow_host_id +assert update_route["shadow_group_id"] == shadow_group_id +assert any(item["route_id"] == route_id for item in list_routes) +assert any(item["public_model"] == public_model and item["shadow_model"] == shadow_model for item in route_models) + +summary = { + "group_id": group_id, + "route_id": route_id, + "public_model": public_model, + "shadow_model": shadow_model, + "shadow_host_id": shadow_host_id, + "shadow_group_id": shadow_group_id, + "checks": { + "group_created": True, + "group_updated": True, + "route_created": True, + "route_updated": True, + "route_model_created": True, + "route_model_listed": True, + }, +} +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY + +cat "$ARTIFACT_DIR/10-summary.json" diff --git a/scripts/acceptance/verify_route_data_plane.sh b/scripts/acceptance/verify_route_data_plane.sh new file mode 100755 index 00000000..d5ef5aeb --- /dev/null +++ b/scripts/acceptance/verify_route_data_plane.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck disable=SC1091 +source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh" + +CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}" +TS="${TS:-$(timestamp_token)}" +ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_data_plane}" + +GROUP_ID="${GROUP_ID:-p2t4-dp-${TS}}" +ROUTE_ID="${ROUTE_ID:-primary-${TS}}" +PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}" +SHADOW_MODEL="${SHADOW_MODEL:-gpt-5.4}" +SHADOW_HOST_ID="${SHADOW_HOST_ID:?SHADOW_HOST_ID required}" +SHADOW_GROUP_ID="${SHADOW_GROUP_ID:?SHADOW_GROUP_ID required}" +REQUEST_ID="${REQUEST_ID:-req-p2t4-dp-${TS}}" +SUBJECT_ID="${SUBJECT_ID:-conv-p2t4-dp-${TS}}" +SUBSCRIPTION_USER_ID="${SUBSCRIPTION_USER_ID:-}" +GATEWAY_API_KEY="${GATEWAY_API_KEY:-}" + +if [[ -z "$SUBSCRIPTION_USER_ID" && -z "$GATEWAY_API_KEY" ]]; then + echo "missing data-plane auth: set SUBSCRIPTION_USER_ID or GATEWAY_API_KEY" >&2 + exit 1 +fi + +crm_auth_init +ensure_artifact_dir + +create_group_payload="$(python3 - "$GROUP_ID" <<'PY' +import json, sys +group_id = sys.argv[1] +print(json.dumps({ + "logical_group_id": group_id, + "display_name": f"P2T4 Data Plane {group_id}", + "status": "active", + "description": "P2-T4 data plane verification group", + "route_policy": "priority", + "sticky_mode": "conversation_preferred", + "conversation_ttl_seconds": 1200, + "user_model_ttl_seconds": 600, + "failover_threshold": 2, + "cooldown_seconds": 300, +}, ensure_ascii=False)) +PY +)" +save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")" +save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")" +save_json 03-create-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "{\"route_id\":\"$ROUTE_ID\",\"name\":\"Primary $ROUTE_ID\",\"status\":\"active\",\"priority\":10,\"weight\":100,\"shadow_group_id\":\"$SHADOW_GROUP_ID\",\"shadow_host_id\":\"$SHADOW_HOST_ID\",\"upstream_base_url_hint\":\"https://real-shadow.example/v1\"}")" +save_json 04-add-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"shadow_model\":\"$SHADOW_MODEL\",\"status\":\"active\"}")" + +chat_payload="$(python3 - "$GROUP_ID" "$PUBLIC_MODEL" "$REQUEST_ID" "$SUBJECT_ID" "$SUBSCRIPTION_USER_ID" "$GATEWAY_API_KEY" <<'PY' +import json, sys +logical_group_id, model, request_id, subject_id, subscription_user_id, gateway_api_key = sys.argv[1:7] +payload = { + "logical_group_id": logical_group_id, + "model": model, + "request_id": request_id, + "scope": "conversation", + "subject_id": subject_id, + "messages": [{"role": "user", "content": "ping"}], + "sync": True, +} +if subscription_user_id: + payload["subscription_user_id"] = subscription_user_id +if gateway_api_key: + payload["gateway_api_key"] = gateway_api_key +print(json.dumps(payload, ensure_ascii=False)) +PY +)" +save_json 05-route-chat "$(crm_curl_json POST "/api/routing/chat/completions" "$chat_payload")" +save_json 06-decision-logs "$(crm_curl_json GET "/api/routing/logs/decisions?request_id=$REQUEST_ID&limit=5")" + +python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$ROUTE_ID" "$PUBLIC_MODEL" "$SHADOW_MODEL" "$SHADOW_HOST_ID" "$SHADOW_GROUP_ID" "$REQUEST_ID" "$SUBSCRIPTION_USER_ID" "$GATEWAY_API_KEY" >"$ARTIFACT_DIR/07-summary.json" <<'PY' +import json +import sys +from pathlib import Path + +( + art_dir, + group_id, + route_id, + public_model, + shadow_model, + shadow_host_id, + shadow_group_id, + request_id, + subscription_user_id, + gateway_api_key, +) = sys.argv[1:11] +art = Path(art_dir) +chat = json.loads((art / "05-route-chat.json").read_text()) +logs = json.loads((art / "06-decision-logs.json").read_text())["decision_logs"] + +assert chat["request_id"] == request_id +assert chat["logical_group_id"] == group_id +assert chat["model"] == public_model +assert chat["selected_route"]["route_id"] == route_id +assert chat["selected_route"]["shadow_host_id"] == shadow_host_id +assert chat["selected_route"]["shadow_group_id"] == shadow_group_id +assert chat["selected_route"]["shadow_model"] == shadow_model +assert chat["forward"]["upstream_status"] == 200 +assert logs, logs +assert logs[0]["request_id"] == request_id +assert logs[0]["selected_route_id"] == route_id + +expected_source = "managed_subscription" if subscription_user_id else "provided_gateway_key" +summary = { + "group_id": group_id, + "route_id": route_id, + "request_id": request_id, + "auth_mode": expected_source, + "forward_upstream_status": chat["forward"]["upstream_status"], + "selected_shadow_host_id": chat["selected_route"]["shadow_host_id"], + "selected_shadow_group_id": chat["selected_route"]["shadow_group_id"], + "selected_shadow_model": chat["selected_route"]["shadow_model"], + "effective_gateway_key_source": chat["forward"].get("effective_gateway_key_source"), + "decision_log_count": len(logs), +} +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY + +cat "$ARTIFACT_DIR/07-summary.json" diff --git a/scripts/acceptance/verify_route_health_ui.sh b/scripts/acceptance/verify_route_health_ui.sh new file mode 100755 index 00000000..893fe4be --- /dev/null +++ b/scripts/acceptance/verify_route_health_ui.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck disable=SC1091 +source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh" + +CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}" +ROUTE_HEALTH_PAGE_URL="${ROUTE_HEALTH_PAGE_URL:-https://sub.tksea.top/portal/admin/route-health.html}" +TS="${TS:-$(timestamp_token)}" +ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_health_ui}" + +GROUP_ID="${GROUP_ID:-p2t4-health-${TS}}" +PRIMARY_ROUTE="${PRIMARY_ROUTE:-primary-${TS}}" +FALLBACK_ROUTE="${FALLBACK_ROUTE:-fallback-${TS}}" +FAILING_ROUTE="${FAILING_ROUTE:-failing-${TS}}" +PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}" +REQUEST_ID="${REQUEST_ID:-req-p2t4-health-${TS}}" +SUBJECT_ID="${SUBJECT_ID:-conv-p2t4-health-${TS}}" + +crm_auth_init +ensure_artifact_dir + +curl_status_to_file "$ROUTE_HEALTH_PAGE_URL" "$ARTIFACT_DIR/00-route-health.html" + +create_group_payload="$(python3 - "$GROUP_ID" <<'PY' +import json, sys +group_id = sys.argv[1] +print(json.dumps({ + "logical_group_id": group_id, + "display_name": f"P2T4 Health {group_id}", + "status": "active", + "description": "P2-T4 health verification group", + "route_policy": "priority", + "sticky_mode": "conversation_preferred", + "conversation_ttl_seconds": 1200, + "user_model_ttl_seconds": 600, + "failover_threshold": 2, + "cooldown_seconds": 300, +}, ensure_ascii=False)) +PY +)" +save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")" +save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")" + +create_route() { + local route_id="$1" + local route_name="$2" + local priority="$3" + local shadow_group_id="$4" + local shadow_host_id="$5" + crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" \ + "{\"route_id\":\"$route_id\",\"name\":\"$route_name\",\"status\":\"active\",\"priority\":$priority,\"weight\":100,\"shadow_group_id\":\"$shadow_group_id\",\"shadow_host_id\":\"$shadow_host_id\",\"upstream_base_url_hint\":\"https://$route_id.example/v1\"}" +} + +save_json 03-create-primary-route "$(create_route "$PRIMARY_ROUTE" "Primary Route" 10 "shadow-primary-$TS" "shadow-host-primary-$TS")" +save_json 04-create-fallback-route "$(create_route "$FALLBACK_ROUTE" "Fallback Route" 20 "shadow-fallback-$TS" "shadow-host-fallback-$TS")" +save_json 05-create-failing-route "$(create_route "$FAILING_ROUTE" "Failing Route" 30 "shadow-failing-$TS" "shadow-host-failing-$TS")" + +for route_id in "$PRIMARY_ROUTE" "$FALLBACK_ROUTE" "$FAILING_ROUTE"; do + save_json "route-model-${route_id}" "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$route_id/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"shadow_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")" +done + +save_json 06-set-cooldown "$(crm_curl_json POST "/api/routing/sticky/cooldowns" "{\"route_id\":\"$PRIMARY_ROUTE\",\"reason\":\"degraded\",\"ttl_seconds\":600}")" +save_json 07-set-failure "$(crm_curl_json POST "/api/routing/sticky/route-failures" "{\"route_id\":\"$FAILING_ROUTE\",\"failure_count\":2,\"last_error_class\":\"timeout\",\"ttl_seconds\":600}")" +save_json 08-health-before "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")" +save_json 09-resolve "$(crm_curl_json POST "/api/routing/resolve" "{\"request_id\":\"$REQUEST_ID\",\"logical_group_id\":\"$GROUP_ID\",\"public_model\":\"$PUBLIC_MODEL\",\"scope\":\"conversation\",\"subject_id\":\"$SUBJECT_ID\",\"sync\":true}")" +save_json 10-health-after "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")" +save_json 11-failovers "$(crm_curl_json GET "/api/routing/logs/failovers?request_id=$REQUEST_ID&limit=5")" + +python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$PRIMARY_ROUTE" "$FALLBACK_ROUTE" "$FAILING_ROUTE" "$REQUEST_ID" >"$ARTIFACT_DIR/12-summary.json" <<'PY' +import json +import sys +from pathlib import Path + +art_dir, group_id, primary_route, fallback_route, failing_route, request_id = sys.argv[1:7] +art = Path(art_dir) +page = (art / "00-route-health.html").read_text() +before = json.loads((art / "08-health-before.json").read_text())["route_health"] +resolve = json.loads((art / "09-resolve.json").read_text())["resolve"] +after = json.loads((art / "10-health-after.json").read_text())["route_health"] +failovers = json.loads((art / "11-failovers.json").read_text())["failover_events"] + +def by_id(items): + return {item["route_id"]: item for item in items} + +before_map = by_id(before) +after_map = by_id(after) + +assert "Route Health Admin" in page +assert before_map[primary_route]["runtime_status"] == "cooldown" +assert before_map[failing_route]["runtime_status"] == "failing" +assert resolve["route_id"] == fallback_route +assert resolve["fallback_used"] is True +assert after_map[primary_route]["runtime_status"] == "cooldown" +assert after_map[fallback_route]["runtime_status"] == "healthy" +assert after_map[fallback_route]["recent_failover_count"] >= 1 +assert failovers and failovers[0]["from_route_id"] == primary_route +assert failovers[0]["to_route_id"] == fallback_route +assert failovers[0]["reason"] == "active_cooldown:degraded" + +summary = { + "group_id": group_id, + "request_id": request_id, + "resolve_route_id": resolve["route_id"], + "resolve_fallback_used": resolve["fallback_used"], + "before_statuses": {k: v["runtime_status"] for k, v in before_map.items()}, + "after_statuses": {k: v["runtime_status"] for k, v in after_map.items()}, + "fallback_recent_failover_count": after_map[fallback_route]["recent_failover_count"], +} +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY + +cat "$ARTIFACT_DIR/12-summary.json" diff --git a/scripts/test/test_real_host_scripts.sh b/scripts/test/test_real_host_scripts.sh index edab7ba8..e827abe3 100755 --- a/scripts/test/test_real_host_scripts.sh +++ b/scripts/test/test_real_host_scripts.sh @@ -660,6 +660,258 @@ EOF [[ -f "$sensitive_root/20260522_foo/05-subscription-access-prep.sql" ]] || fail "sql file was not moved to sensitive mirror" } +run_test_verify_route_control_plane_script() { + local tmpdir fakebin artifact_dir stdout_file + tmpdir="$(mktemp -d)" + trap 'rm -rf "$tmpdir"' RETURN + fakebin="$tmpdir/bin" + artifact_dir="$tmpdir/artifacts" + stdout_file="$tmpdir/verify_route_control_plane.stdout.txt" + mkdir -p "$fakebin" "$artifact_dir" + + cat > "$fakebin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +method="GET" +url="" +payload="" +prev="" +for arg in "$@"; do + case "$prev" in + -X) method="$arg"; prev=""; continue ;; + -d|--data) payload="$arg"; prev=""; continue ;; + esac + case "$arg" in + -X|-d|--data) prev="$arg"; continue ;; + http://*|https://*) url="$arg" ;; + esac +done +case "$method $url" in + "POST http://crm.example.com/api/logical-groups") + printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane p2t4-cp-1700000000","status":"active"}}' + ;; + "POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/models") + printf '%s\n' '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}' + ;; + "POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes") + printf '%s\n' '{"logical_group_route":{"route_id":"primary-1700000000","logical_group_id":"p2t4-cp-1700000000","name":"Primary primary-1700000000","status":"active","priority":10,"weight":100,"shadow_group_id":"shadow-group-1700000000","shadow_host_id":"shadow-host-1700000000"}}' + ;; + "POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000/models") + printf '%s\n' '{"logical_group_route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4","status":"active"}}' + ;; + "GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000") + printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane p2t4-cp-1700000000","status":"active","models":[{"public_model":"gpt-5.4"}],"routes":[{"route_id":"primary-1700000000"}]}}' + ;; + "PUT http://crm.example.com/api/logical-groups/p2t4-cp-1700000000") + printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane Updated p2t4-cp-1700000000","status":"active"}}' + ;; + "PUT http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000") + printf '%s\n' '{"logical_group_route":{"route_id":"primary-1700000000","logical_group_id":"p2t4-cp-1700000000","name":"Primary Route Updated","status":"active","priority":12,"weight":80,"shadow_group_id":"shadow-group-1700000000","shadow_host_id":"shadow-host-1700000000"}}' + ;; + "GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes") + printf '%s\n' '{"routes":[{"route_id":"primary-1700000000","weight":80}]}' + ;; + "GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000/models") + printf '%s\n' '{"models":[{"public_model":"gpt-5.4","shadow_model":"gpt-5.4","status":"active"}]}' + ;; + *) + echo "unexpected curl request: $method $url payload=$payload" >&2 + exit 1 + ;; +esac +EOF + chmod +x "$fakebin/curl" + + PATH="$fakebin:$PATH" \ + CRM_BASE="http://crm.example.com" \ + CRM_ADMIN_TOKEN="token" \ + TS="1700000000" \ + ARTIFACT_DIR="$artifact_dir" \ + bash "$ROOT_DIR/scripts/acceptance/verify_route_control_plane.sh" >"$stdout_file" + + local summary stdout_text + summary="$(cat "$artifact_dir/10-summary.json")" + stdout_text="$(cat "$stdout_file")" + assert_contains "$summary" '"group_id": "p2t4-cp-1700000000"' + assert_contains "$summary" '"route_id": "primary-1700000000"' + assert_contains "$summary" '"route_updated": true' + assert_contains "$stdout_text" '"route_model_listed": true' +} + +run_test_verify_route_data_plane_script() { + local tmpdir fakebin artifact_dir stdout_file payload_log + tmpdir="$(mktemp -d)" + trap 'rm -rf "$tmpdir"' RETURN + fakebin="$tmpdir/bin" + artifact_dir="$tmpdir/artifacts" + stdout_file="$tmpdir/verify_route_data_plane.stdout.txt" + payload_log="$tmpdir/payload.log" + mkdir -p "$fakebin" "$artifact_dir" + + cat > "$fakebin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +method="GET" +url="" +payload="" +prev="" +for arg in "$@"; do + case "$prev" in + -X) method="$arg"; prev=""; continue ;; + -d|--data) payload="$arg"; prev=""; continue ;; + esac + case "$arg" in + -X|-d|--data) prev="$arg"; continue ;; + http://*|https://*) url="$arg" ;; + esac +done +printf '%s\n' "$payload" >> "${PAYLOAD_LOG:?missing PAYLOAD_LOG}" +case "$method $url" in + "POST http://crm.example.com/api/logical-groups") + printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-dp-1700000001","display_name":"P2T4 Data Plane p2t4-dp-1700000001","status":"active"}}' + ;; + "POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/models") + printf '%s\n' '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}' + ;; + "POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/routes") + printf '%s\n' '{"logical_group_route":{"route_id":"primary-1700000001","shadow_group_id":"shadow-group-9","shadow_host_id":"shadow-host-real"}}' + ;; + "POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/routes/primary-1700000001/models") + printf '%s\n' '{"logical_group_route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4"}}' + ;; + "POST http://crm.example.com/api/routing/chat/completions") + printf '%s\n' '{"request_id":"req-p2t4-dp-1700000001","logical_group_id":"p2t4-dp-1700000001","model":"gpt-5.4","selected_route":{"route_id":"primary-1700000001","shadow_host_id":"shadow-host-real","shadow_group_id":"shadow-group-9","shadow_model":"gpt-5.4"},"forward":{"upstream_status":200,"effective_gateway_key_source":"managed_subscription"}}' + ;; + "GET http://crm.example.com/api/routing/logs/decisions?request_id=req-p2t4-dp-1700000001&limit=5") + printf '%s\n' '{"decision_logs":[{"request_id":"req-p2t4-dp-1700000001","selected_route_id":"primary-1700000001"}]}' + ;; + *) + echo "unexpected curl request: $method $url payload=$payload" >&2 + exit 1 + ;; +esac +EOF + chmod +x "$fakebin/curl" + + PATH="$fakebin:$PATH" \ + PAYLOAD_LOG="$payload_log" \ + CRM_BASE="http://crm.example.com" \ + CRM_ADMIN_TOKEN="token" \ + TS="1700000001" \ + SHADOW_HOST_ID="shadow-host-real" \ + SHADOW_GROUP_ID="shadow-group-9" \ + SUBSCRIPTION_USER_ID="36" \ + ARTIFACT_DIR="$artifact_dir" \ + bash "$ROOT_DIR/scripts/acceptance/verify_route_data_plane.sh" >"$stdout_file" + + local summary payloads + summary="$(cat "$artifact_dir/07-summary.json")" + payloads="$(cat "$payload_log")" + assert_contains "$summary" '"forward_upstream_status": 200' + assert_contains "$summary" '"effective_gateway_key_source": "managed_subscription"' + assert_contains "$payloads" '"subscription_user_id": "36"' +} + +run_test_verify_route_health_ui_script() { + local tmpdir fakebin artifact_dir stdout_file + tmpdir="$(mktemp -d)" + trap 'rm -rf "$tmpdir"' RETURN + fakebin="$tmpdir/bin" + artifact_dir="$tmpdir/artifacts" + stdout_file="$tmpdir/verify_route_health_ui.stdout.txt" + mkdir -p "$fakebin" "$artifact_dir" + + cat > "$fakebin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +method="GET" +url="" +payload="" +output_file="" +prev="" +for arg in "$@"; do + case "$prev" in + -X) method="$arg"; prev=""; continue ;; + -d|--data) payload="$arg"; prev=""; continue ;; + -o) output_file="$arg"; prev=""; continue ;; + esac + case "$arg" in + -X|-d|--data|-o) prev="$arg"; continue ;; + http://*|https://*) url="$arg" ;; + esac +done +write_body() { + local body="$1" + if [[ -n "$output_file" ]]; then + printf '%s\n' "$body" > "$output_file" + else + printf '%s\n' "$body" + fi +} +case "$method $url" in + "GET http://portal.example.com/route-health.html") + write_body 'Route Health AdminRoute Health Admin' + ;; + "POST http://crm.example.com/api/logical-groups") + write_body '{"logical_group":{"logical_group_id":"p2t4-health-1700000002","status":"active"}}' + ;; + "POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/models") + write_body '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}' + ;; + "POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes") + if [[ "$payload" == *'"route_id":"primary-1700000002"'* ]]; then + write_body '{"logical_group_route":{"route_id":"primary-1700000002"}}' + elif [[ "$payload" == *'"route_id":"fallback-1700000002"'* ]]; then + write_body '{"logical_group_route":{"route_id":"fallback-1700000002"}}' + else + write_body '{"logical_group_route":{"route_id":"failing-1700000002"}}' + fi + ;; + "POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/primary-1700000002/models"|"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/fallback-1700000002/models"|"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/failing-1700000002/models") + write_body '{"logical_group_route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4"}}' + ;; + "POST http://crm.example.com/api/routing/sticky/cooldowns") + write_body '{"route_cooldown":{"route_id":"primary-1700000002","reason":"degraded"}}' + ;; + "POST http://crm.example.com/api/routing/sticky/route-failures") + write_body '{"route_failure":{"route_id":"failing-1700000002","failure_count":2,"last_error_class":"timeout"}}' + ;; + "GET http://crm.example.com/api/routing/routes/health?logical_group_id=p2t4-health-1700000002") + if [[ ! -f /tmp/p2t4-health-switch ]]; then + write_body '{"route_health":[{"route_id":"primary-1700000002","runtime_status":"cooldown"},{"route_id":"fallback-1700000002","runtime_status":"healthy","recent_failover_count":0},{"route_id":"failing-1700000002","runtime_status":"failing"}]}' + else + write_body '{"route_health":[{"route_id":"primary-1700000002","runtime_status":"cooldown"},{"route_id":"fallback-1700000002","runtime_status":"healthy","recent_failover_count":1,"last_selected_at":"2026-05-29T12:00:00Z"},{"route_id":"failing-1700000002","runtime_status":"failing"}]}' + fi + ;; + "POST http://crm.example.com/api/routing/resolve") + : > /tmp/p2t4-health-switch + write_body '{"resolve":{"request_id":"req-p2t4-health-1700000002","route_id":"fallback-1700000002","fallback_used":true}}' + ;; + "GET http://crm.example.com/api/routing/logs/failovers?request_id=req-p2t4-health-1700000002&limit=5") + write_body '{"failover_events":[{"from_route_id":"primary-1700000002","to_route_id":"fallback-1700000002","reason":"active_cooldown:degraded"}]}' + ;; + *) + echo "unexpected curl request: $method $url payload=$payload" >&2 + exit 1 + ;; +esac +EOF + chmod +x "$fakebin/curl" + + PATH="$fakebin:$PATH" \ + CRM_BASE="http://crm.example.com" \ + CRM_ADMIN_TOKEN="token" \ + ROUTE_HEALTH_PAGE_URL="http://portal.example.com/route-health.html" \ + TS="1700000002" \ + ARTIFACT_DIR="$artifact_dir" \ + bash "$ROOT_DIR/scripts/acceptance/verify_route_health_ui.sh" >"$stdout_file" + + local summary + summary="$(cat "$artifact_dir/12-summary.json")" + assert_contains "$summary" '"resolve_route_id": "fallback-1700000002"' + assert_contains "$summary" '"fallback_recent_failover_count": 1' +} + run_test_remote43_patched_stack_renderers() { # shellcheck disable=SC1091 source "$ROOT_DIR/scripts/deploy/remote43_patched_stack_lib.sh" @@ -784,6 +1036,9 @@ run_test_real_host_acceptance_after_import_hook run_test_check_deepseek_completion_split run_test_import_remote43_provider_subscription_prep run_test_migrate_historical_artifacts +run_test_verify_route_control_plane_script +run_test_verify_route_data_plane_script +run_test_verify_route_health_ui_script run_test_remote43_patched_stack_renderers run_test_setup_remote43_patched_stack_dry_run