niuniu/sub2api-cn-relay-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(routing): add route acceptance matrix scripts

Browse Source
This commit is contained in:
phamnazage-jpg
2026-05-29 13:50:16 +08:00
parent 112ed67f6b
commit 949134005e
7 changed files with 994 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
docs/ROUTE_ACCEPTANCE_MATRIX.md Normal file
View File

@@ -0,0 +1,165 @@
# Route Acceptance Matrix
日期：2026-05-29
## 目标
把 Phase 2 已经验证过的三类能力，收敛成固定入口和固定证据格式：
1. 控制面
2. 运行态 / 健康视图
3. 数据面
当前统一入口：
- `scripts/acceptance/verify_route_control_plane.sh`
- `scripts/acceptance/verify_route_health_ui.sh`
- `scripts/acceptance/verify_route_data_plane.sh`
- `scripts/acceptance/verify_route_acceptance_matrix.sh`
其中 `verify_route_acceptance_matrix.sh` 是总入口，会按顺序执行前三个脚本并汇总 `summary.json`。
## 验收矩阵
### 1. 控制面
脚本：
- `scripts/acceptance/verify_route_control_plane.sh`
验证范围：
- 创建 `logical_group`
- 创建 `logical_group_model`
- 创建 `route`
- 创建 `route_model`
- `GET /api/logical-groups/{group_id}`
- `PUT /api/logical-groups/{group_id}`
- `PUT /api/logical-groups/{group_id}/routes/{route_id}`
- `GET /api/logical-groups/{group_id}/routes`
- `GET /api/logical-groups/{group_id}/routes/{route_id}/models`
最小输入：
- `CRM_BASE`
- `CRM_ADMIN_TOKEN`
主要产物：
- `01-create-group.json`
- `03-create-route.json`
- `06-update-group.json`
- `07-update-route.json`
- `10-summary.json`
### 2. 健康视图 / 运行态
脚本：
- `scripts/acceptance/verify_route_health_ui.sh`
验证范围：
- 公网 `route-health.html` 页面可达
- `POST /api/routing/sticky/cooldowns`
- `POST /api/routing/sticky/route-failures`
- `GET /api/routing/routes/health`
- `POST /api/routing/resolve`
- `GET /api/routing/logs/failovers`
最小输入：
- `CRM_BASE`
- `CRM_ADMIN_TOKEN`
- 可选 `ROUTE_HEALTH_PAGE_URL`
主要产物：
- `00-route-health.html`
- `08-health-before.json`
- `09-resolve.json`
- `10-health-after.json`
- `11-failovers.json`
- `12-summary.json`
### 3. 数据面
脚本：
- `scripts/acceptance/verify_route_data_plane.sh`
验证范围：
- 创建临时 `logical_group / route / route_model`
- `POST /api/routing/chat/completions`
- `GET /api/routing/logs/decisions`
最小输入：
- `CRM_BASE`
- `CRM_ADMIN_TOKEN`
- `SHADOW_HOST_ID`
- `SHADOW_GROUP_ID`
- 二选一：
- `SUBSCRIPTION_USER_ID`
- `GATEWAY_API_KEY`
补充说明：
- 若传 `SUBSCRIPTION_USER_ID`，期望 `effective_gateway_key_source=managed_subscription`
- 若传 `GATEWAY_API_KEY`，脚本仍可跑通，但这不覆盖“自动供给 managed key”场景
主要产物：
- `05-route-chat.json`
- `06-decision-logs.json`
- `07-summary.json`
## 总入口
脚本：
- `scripts/acceptance/verify_route_acceptance_matrix.sh`
它会创建：
```text
artifacts/phase2-routing-matrix/<timestamp>_route_matrix/
control_plane/
health_ui/
data_plane/
summary.json
```
`summary.json` 只保留最核心的摘要字段：
- `control_plane_group_id`
- `health_ui_group_id`
- `data_plane_group_id`
- `data_plane_request_id`
- `data_plane_upstream_status`
- `health_ui_resolve_route_id`
## remote43 推荐执行方式
如果要避免把真实 `CRM_ADMIN_TOKEN` 带回本地，推荐直接在 `remote43` 上执行：
```bash
cd /home/ubuntu/sub2api-kimi-patched-auto2-20260525_18169
set -a
. ./.env.crm
set +a
cd /home/ubuntu/sub2api-cn-relay-manager-git-current
CRM_BASE="https://sub.tksea.top/portal-admin-api" \
SHADOW_HOST_ID="<real-shadow-host-id>" \
SHADOW_GROUP_ID="<real-shadow-group-id>" \
SUBSCRIPTION_USER_ID="<managed-user-id>" \
bash ./scripts/acceptance/verify_route_acceptance_matrix.sh
```
## 当前边界
- 脚本只负责留证据和判断通过/失败，不负责清理临时 `logical_group`
- 数据面脚本当前仍依赖调用方提供真实 `shadow_host_id / shadow_group_id`
- 尚未覆盖“同公开模型双线路主备”的未来策略矩阵

131
scripts/acceptance/route_acceptance_lib.sh Normal file
View File

@@ -0,0 +1,131 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROUTE_MATRIX_ROOT="${ROUTE_MATRIX_ROOT:-$ROOT_DIR/artifacts/phase2-routing-matrix}"
require_var() {
local name="$1"
if [[ -z "${!name:-}" ]]; then
echo "missing required env: $name" >&2
exit 1
fi
}
ensure_artifact_dir() {
mkdir -p "$ARTIFACT_DIR"
}
save_json() {
local name="$1"
local payload="$2"
ensure_artifact_dir
printf '%s\n' "$payload" > "$ARTIFACT_DIR/$name.json"
}
save_text() {
local name="$1"
local payload="$2"
ensure_artifact_dir
printf '%s\n' "$payload" > "$ARTIFACT_DIR/$name"
}
json_get_file() {
local file="$1"
local key="$2"
python3 - "$file" "$key" <<'PY'
import json
import sys
file_path, key = sys.argv[1:3]
with open(file_path, 'r', encoding='utf-8') as fh:
value = json.load(fh)
for part in key.split('.'):
if isinstance(value, dict):
value = value.get(part)
else:
value = None
break
if value is None:
raise SystemExit(2)
if isinstance(value, (dict, list)):
print(json.dumps(value, ensure_ascii=False))
else:
print(value)
PY
}
crm_auth_init() {
require_var CRM_BASE
if [[ -n "${CRM_ADMIN_TOKEN:-}" ]]; then
crm_token="$CRM_ADMIN_TOKEN"
crm_cookie_jar=""
return 0
fi
if [[ -n "${CRM_COOKIE_JAR:-}" ]]; then
crm_cookie_jar="$CRM_COOKIE_JAR"
else
crm_cookie_jar="$(mktemp /tmp/route-matrix-cookie.XXXXXX.jar)"
fi
rm -f "$crm_cookie_jar"
require_var CRM_ADMIN_USERNAME
require_var CRM_ADMIN_PASSWORD
local login_payload
login_payload="$(python3 - "$CRM_ADMIN_USERNAME" "$CRM_ADMIN_PASSWORD" <<'PY'
import json
import sys
username, password = sys.argv[1:3]
print(json.dumps({"username": username, "password": password}, ensure_ascii=False))
PY
)"
curl -fsS \
-c "$crm_cookie_jar" \
-b "$crm_cookie_jar" \
-H 'Content-Type: application/json' \
-X POST \
"${CRM_BASE%/}/api/admin/session/login" \
-d "$login_payload" > /dev/null
crm_token=""
}
crm_curl_json() {
local method="$1"
local path="$2"
local payload="${3:-}"
local -a curl_args
curl_args=(-fsS -X "$method")
if [[ -n "${crm_token:-}" ]]; then
curl_args+=(-H "Authorization: Bearer $crm_token")
elif [[ -n "${crm_cookie_jar:-}" ]]; then
curl_args+=(-b "$crm_cookie_jar" -c "$crm_cookie_jar")
else
echo "missing CRM auth: set CRM_ADMIN_TOKEN or CRM_ADMIN_USERNAME/CRM_ADMIN_PASSWORD" >&2
exit 2
fi
if [[ -n "$payload" ]]; then
curl_args+=(
-H 'Content-Type: application/json'
"${CRM_BASE%/}${path}"
-d "$payload"
)
else
curl_args+=("${CRM_BASE%/}${path}")
fi
curl "${curl_args[@]}"
}
curl_status_to_file() {
local url="$1"
local file="$2"
ensure_artifact_dir
curl -fsS "$url" -o "$file"
}
timestamp_token() {
date +%s
}

47
scripts/acceptance/verify_route_acceptance_matrix.sh Executable file
View File

@@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
# shellcheck disable=SC1091
source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"
CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
TS="${TS:-$(timestamp_token)}"
MATRIX_DIR="${MATRIX_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_matrix}"
mkdir -p "$MATRIX_DIR"
run_step() {
local name="$1"
shift
echo "==> $name"
ARTIFACT_DIR="$MATRIX_DIR/$name" "$@"
}
run_step control_plane bash "$ROOT_DIR/scripts/acceptance/verify_route_control_plane.sh"
run_step health_ui bash "$ROOT_DIR/scripts/acceptance/verify_route_health_ui.sh"
run_step data_plane bash "$ROOT_DIR/scripts/acceptance/verify_route_data_plane.sh"
python3 - "$MATRIX_DIR" >"$MATRIX_DIR/summary.json" <<'PY'
import json
import sys
from pathlib import Path
matrix_dir = Path(sys.argv[1])
control = json.loads((matrix_dir / "control_plane" / "10-summary.json").read_text())
health = json.loads((matrix_dir / "health_ui" / "12-summary.json").read_text())
data = json.loads((matrix_dir / "data_plane" / "07-summary.json").read_text())
summary = {
"matrix_dir": str(matrix_dir),
"control_plane_group_id": control["group_id"],
"health_ui_group_id": health["group_id"],
"data_plane_group_id": data["group_id"],
"data_plane_request_id": data["request_id"],
"data_plane_upstream_status": data["forward_upstream_status"],
"health_ui_resolve_route_id": health["resolve_route_id"],
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
cat "$MATRIX_DIR/summary.json"

158
scripts/acceptance/verify_route_control_plane.sh Executable file
View File

@@ -0,0 +1,158 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
# shellcheck disable=SC1091
source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"
CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
TS="${TS:-$(timestamp_token)}"
ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_control_plane}"
GROUP_ID="${GROUP_ID:-p2t4-cp-${TS}}"
ROUTE_ID="${ROUTE_ID:-primary-${TS}}"
PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}"
SHADOW_MODEL="${SHADOW_MODEL:-gpt-5.4}"
SHADOW_HOST_ID="${SHADOW_HOST_ID:-shadow-host-${TS}}"
SHADOW_GROUP_ID="${SHADOW_GROUP_ID:-shadow-group-${TS}}"
crm_auth_init
ensure_artifact_dir
create_group_payload="$(python3 - "$GROUP_ID" <<'PY'
import json, sys
group_id = sys.argv[1]
print(json.dumps({
"logical_group_id": group_id,
"display_name": f"P2T4 Control Plane {group_id}",
"status": "active",
"description": "P2-T4 control plane verification group",
"route_policy": "priority",
"sticky_mode": "conversation_preferred",
"conversation_ttl_seconds": 1200,
"user_model_ttl_seconds": 600,
"failover_threshold": 2,
"cooldown_seconds": 300,
}, ensure_ascii=False))
PY
)"
save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")"
add_model_payload="$(python3 - "$PUBLIC_MODEL" <<'PY'
import json, sys
print(json.dumps({"public_model": sys.argv[1], "status": "active"}, ensure_ascii=False))
PY
)"
save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "$add_model_payload")"
create_route_payload="$(python3 - "$ROUTE_ID" "$SHADOW_GROUP_ID" "$SHADOW_HOST_ID" <<'PY'
import json, sys
route_id, shadow_group_id, shadow_host_id = sys.argv[1:4]
print(json.dumps({
"route_id": route_id,
"name": f"Primary {route_id}",
"status": "active",
"priority": 10,
"weight": 100,
"shadow_group_id": shadow_group_id,
"shadow_host_id": shadow_host_id,
"upstream_base_url_hint": "https://primary.example/v1",
}, ensure_ascii=False))
PY
)"
save_json 03-create-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "$create_route_payload")"
add_route_model_payload="$(python3 - "$PUBLIC_MODEL" "$SHADOW_MODEL" <<'PY'
import json, sys
public_model, shadow_model = sys.argv[1:3]
print(json.dumps({
"public_model": public_model,
"shadow_model": shadow_model,
"status": "active",
}, ensure_ascii=False))
PY
)"
save_json 04-add-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID/models" "$add_route_model_payload")"
save_json 05-get-group "$(crm_curl_json GET "/api/logical-groups/$GROUP_ID")"
update_group_payload="$(python3 - "$GROUP_ID" <<'PY'
import json, sys
group_id = sys.argv[1]
print(json.dumps({
"display_name": f"P2T4 Control Plane Updated {group_id}",
"status": "active",
"description": "P2-T4 control plane verification group updated",
"route_policy": "priority",
"sticky_mode": "conversation_preferred",
"conversation_ttl_seconds": 1500,
"user_model_ttl_seconds": 900,
"failover_threshold": 2,
"cooldown_seconds": 360,
}, ensure_ascii=False))
PY
)"
save_json 06-update-group "$(crm_curl_json PUT "/api/logical-groups/$GROUP_ID" "$update_group_payload")"
update_route_payload="$(python3 - "$SHADOW_GROUP_ID" "$SHADOW_HOST_ID" <<'PY'
import json, sys
shadow_group_id, shadow_host_id = sys.argv[1:3]
print(json.dumps({
"name": "Primary Route Updated",
"status": "active",
"priority": 12,
"weight": 80,
"shadow_group_id": shadow_group_id,
"shadow_host_id": shadow_host_id,
"upstream_base_url_hint": "https://primary-updated.example/v1",
"cooldown_until": "",
}, ensure_ascii=False))
PY
)"
save_json 07-update-route "$(crm_curl_json PUT "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID" "$update_route_payload")"
save_json 08-list-routes "$(crm_curl_json GET "/api/logical-groups/$GROUP_ID/routes")"
save_json 09-list-route-models "$(crm_curl_json GET "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID/models")"
python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$ROUTE_ID" "$PUBLIC_MODEL" "$SHADOW_MODEL" "$SHADOW_HOST_ID" "$SHADOW_GROUP_ID" >"$ARTIFACT_DIR/10-summary.json" <<'PY'
import json
import sys
from pathlib import Path
art_dir, group_id, route_id, public_model, shadow_model, shadow_host_id, shadow_group_id = sys.argv[1:8]
art = Path(art_dir)
create_group = json.loads((art / "01-create-group.json").read_text())["logical_group"]
update_group = json.loads((art / "06-update-group.json").read_text())["logical_group"]
update_route = json.loads((art / "07-update-route.json").read_text())["logical_group_route"]
list_routes = json.loads((art / "08-list-routes.json").read_text())["routes"]
route_models = json.loads((art / "09-list-route-models.json").read_text())["models"]
assert create_group["logical_group_id"] == group_id
assert update_group["display_name"].startswith("P2T4 Control Plane Updated")
assert update_route["route_id"] == route_id
assert update_route["weight"] == 80
assert update_route["shadow_host_id"] == shadow_host_id
assert update_route["shadow_group_id"] == shadow_group_id
assert any(item["route_id"] == route_id for item in list_routes)
assert any(item["public_model"] == public_model and item["shadow_model"] == shadow_model for item in route_models)
summary = {
"group_id": group_id,
"route_id": route_id,
"public_model": public_model,
"shadow_model": shadow_model,
"shadow_host_id": shadow_host_id,
"shadow_group_id": shadow_group_id,
"checks": {
"group_created": True,
"group_updated": True,
"route_created": True,
"route_updated": True,
"route_model_created": True,
"route_model_listed": True,
},
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
cat "$ARTIFACT_DIR/10-summary.json"

124
scripts/acceptance/verify_route_data_plane.sh Executable file
View File

@@ -0,0 +1,124 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
# shellcheck disable=SC1091
source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"
CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
TS="${TS:-$(timestamp_token)}"
ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_data_plane}"
GROUP_ID="${GROUP_ID:-p2t4-dp-${TS}}"
ROUTE_ID="${ROUTE_ID:-primary-${TS}}"
PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}"
SHADOW_MODEL="${SHADOW_MODEL:-gpt-5.4}"
SHADOW_HOST_ID="${SHADOW_HOST_ID:?SHADOW_HOST_ID required}"
SHADOW_GROUP_ID="${SHADOW_GROUP_ID:?SHADOW_GROUP_ID required}"
REQUEST_ID="${REQUEST_ID:-req-p2t4-dp-${TS}}"
SUBJECT_ID="${SUBJECT_ID:-conv-p2t4-dp-${TS}}"
SUBSCRIPTION_USER_ID="${SUBSCRIPTION_USER_ID:-}"
GATEWAY_API_KEY="${GATEWAY_API_KEY:-}"
if [[ -z "$SUBSCRIPTION_USER_ID" && -z "$GATEWAY_API_KEY" ]]; then
echo "missing data-plane auth: set SUBSCRIPTION_USER_ID or GATEWAY_API_KEY" >&2
exit 1
fi
crm_auth_init
ensure_artifact_dir
create_group_payload="$(python3 - "$GROUP_ID" <<'PY'
import json, sys
group_id = sys.argv[1]
print(json.dumps({
"logical_group_id": group_id,
"display_name": f"P2T4 Data Plane {group_id}",
"status": "active",
"description": "P2-T4 data plane verification group",
"route_policy": "priority",
"sticky_mode": "conversation_preferred",
"conversation_ttl_seconds": 1200,
"user_model_ttl_seconds": 600,
"failover_threshold": 2,
"cooldown_seconds": 300,
}, ensure_ascii=False))
PY
)"
save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")"
save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")"
save_json 03-create-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "{\"route_id\":\"$ROUTE_ID\",\"name\":\"Primary $ROUTE_ID\",\"status\":\"active\",\"priority\":10,\"weight\":100,\"shadow_group_id\":\"$SHADOW_GROUP_ID\",\"shadow_host_id\":\"$SHADOW_HOST_ID\",\"upstream_base_url_hint\":\"https://real-shadow.example/v1\"}")"
save_json 04-add-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$ROUTE_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"shadow_model\":\"$SHADOW_MODEL\",\"status\":\"active\"}")"
chat_payload="$(python3 - "$GROUP_ID" "$PUBLIC_MODEL" "$REQUEST_ID" "$SUBJECT_ID" "$SUBSCRIPTION_USER_ID" "$GATEWAY_API_KEY" <<'PY'
import json, sys
logical_group_id, model, request_id, subject_id, subscription_user_id, gateway_api_key = sys.argv[1:7]
payload = {
"logical_group_id": logical_group_id,
"model": model,
"request_id": request_id,
"scope": "conversation",
"subject_id": subject_id,
"messages": [{"role": "user", "content": "ping"}],
"sync": True,
}
if subscription_user_id:
payload["subscription_user_id"] = subscription_user_id
if gateway_api_key:
payload["gateway_api_key"] = gateway_api_key
print(json.dumps(payload, ensure_ascii=False))
PY
)"
save_json 05-route-chat "$(crm_curl_json POST "/api/routing/chat/completions" "$chat_payload")"
save_json 06-decision-logs "$(crm_curl_json GET "/api/routing/logs/decisions?request_id=$REQUEST_ID&limit=5")"
python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$ROUTE_ID" "$PUBLIC_MODEL" "$SHADOW_MODEL" "$SHADOW_HOST_ID" "$SHADOW_GROUP_ID" "$REQUEST_ID" "$SUBSCRIPTION_USER_ID" "$GATEWAY_API_KEY" >"$ARTIFACT_DIR/07-summary.json" <<'PY'
import json
import sys
from pathlib import Path
(
art_dir,
group_id,
route_id,
public_model,
shadow_model,
shadow_host_id,
shadow_group_id,
request_id,
subscription_user_id,
gateway_api_key,
) = sys.argv[1:11]
art = Path(art_dir)
chat = json.loads((art / "05-route-chat.json").read_text())
logs = json.loads((art / "06-decision-logs.json").read_text())["decision_logs"]
assert chat["request_id"] == request_id
assert chat["logical_group_id"] == group_id
assert chat["model"] == public_model
assert chat["selected_route"]["route_id"] == route_id
assert chat["selected_route"]["shadow_host_id"] == shadow_host_id
assert chat["selected_route"]["shadow_group_id"] == shadow_group_id
assert chat["selected_route"]["shadow_model"] == shadow_model
assert chat["forward"]["upstream_status"] == 200
assert logs, logs
assert logs[0]["request_id"] == request_id
assert logs[0]["selected_route_id"] == route_id
expected_source = "managed_subscription" if subscription_user_id else "provided_gateway_key"
summary = {
"group_id": group_id,
"route_id": route_id,
"request_id": request_id,
"auth_mode": expected_source,
"forward_upstream_status": chat["forward"]["upstream_status"],
"selected_shadow_host_id": chat["selected_route"]["shadow_host_id"],
"selected_shadow_group_id": chat["selected_route"]["shadow_group_id"],
"selected_shadow_model": chat["selected_route"]["shadow_model"],
"effective_gateway_key_source": chat["forward"].get("effective_gateway_key_source"),
"decision_log_count": len(logs),
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
cat "$ARTIFACT_DIR/07-summary.json"

114
scripts/acceptance/verify_route_health_ui.sh Executable file
View File

@@ -0,0 +1,114 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
# shellcheck disable=SC1091
source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"
CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
ROUTE_HEALTH_PAGE_URL="${ROUTE_HEALTH_PAGE_URL:-https://sub.tksea.top/portal/admin/route-health.html}"
TS="${TS:-$(timestamp_token)}"
ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_health_ui}"
GROUP_ID="${GROUP_ID:-p2t4-health-${TS}}"
PRIMARY_ROUTE="${PRIMARY_ROUTE:-primary-${TS}}"
FALLBACK_ROUTE="${FALLBACK_ROUTE:-fallback-${TS}}"
FAILING_ROUTE="${FAILING_ROUTE:-failing-${TS}}"
PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}"
REQUEST_ID="${REQUEST_ID:-req-p2t4-health-${TS}}"
SUBJECT_ID="${SUBJECT_ID:-conv-p2t4-health-${TS}}"
crm_auth_init
ensure_artifact_dir
curl_status_to_file "$ROUTE_HEALTH_PAGE_URL" "$ARTIFACT_DIR/00-route-health.html"
create_group_payload="$(python3 - "$GROUP_ID" <<'PY'
import json, sys
group_id = sys.argv[1]
print(json.dumps({
"logical_group_id": group_id,
"display_name": f"P2T4 Health {group_id}",
"status": "active",
"description": "P2-T4 health verification group",
"route_policy": "priority",
"sticky_mode": "conversation_preferred",
"conversation_ttl_seconds": 1200,
"user_model_ttl_seconds": 600,
"failover_threshold": 2,
"cooldown_seconds": 300,
}, ensure_ascii=False))
PY
)"
save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")"
save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")"
create_route() {
local route_id="$1"
local route_name="$2"
local priority="$3"
local shadow_group_id="$4"
local shadow_host_id="$5"
crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" \
"{\"route_id\":\"$route_id\",\"name\":\"$route_name\",\"status\":\"active\",\"priority\":$priority,\"weight\":100,\"shadow_group_id\":\"$shadow_group_id\",\"shadow_host_id\":\"$shadow_host_id\",\"upstream_base_url_hint\":\"https://$route_id.example/v1\"}"
}
save_json 03-create-primary-route "$(create_route "$PRIMARY_ROUTE" "Primary Route" 10 "shadow-primary-$TS" "shadow-host-primary-$TS")"
save_json 04-create-fallback-route "$(create_route "$FALLBACK_ROUTE" "Fallback Route" 20 "shadow-fallback-$TS" "shadow-host-fallback-$TS")"
save_json 05-create-failing-route "$(create_route "$FAILING_ROUTE" "Failing Route" 30 "shadow-failing-$TS" "shadow-host-failing-$TS")"
for route_id in "$PRIMARY_ROUTE" "$FALLBACK_ROUTE" "$FAILING_ROUTE"; do
save_json "route-model-${route_id}" "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$route_id/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"shadow_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")"
done
save_json 06-set-cooldown "$(crm_curl_json POST "/api/routing/sticky/cooldowns" "{\"route_id\":\"$PRIMARY_ROUTE\",\"reason\":\"degraded\",\"ttl_seconds\":600}")"
save_json 07-set-failure "$(crm_curl_json POST "/api/routing/sticky/route-failures" "{\"route_id\":\"$FAILING_ROUTE\",\"failure_count\":2,\"last_error_class\":\"timeout\",\"ttl_seconds\":600}")"
save_json 08-health-before "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")"
save_json 09-resolve "$(crm_curl_json POST "/api/routing/resolve" "{\"request_id\":\"$REQUEST_ID\",\"logical_group_id\":\"$GROUP_ID\",\"public_model\":\"$PUBLIC_MODEL\",\"scope\":\"conversation\",\"subject_id\":\"$SUBJECT_ID\",\"sync\":true}")"
save_json 10-health-after "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")"
save_json 11-failovers "$(crm_curl_json GET "/api/routing/logs/failovers?request_id=$REQUEST_ID&limit=5")"
python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$PRIMARY_ROUTE" "$FALLBACK_ROUTE" "$FAILING_ROUTE" "$REQUEST_ID" >"$ARTIFACT_DIR/12-summary.json" <<'PY'
import json
import sys
from pathlib import Path
art_dir, group_id, primary_route, fallback_route, failing_route, request_id = sys.argv[1:7]
art = Path(art_dir)
page = (art / "00-route-health.html").read_text()
before = json.loads((art / "08-health-before.json").read_text())["route_health"]
resolve = json.loads((art / "09-resolve.json").read_text())["resolve"]
after = json.loads((art / "10-health-after.json").read_text())["route_health"]
failovers = json.loads((art / "11-failovers.json").read_text())["failover_events"]
def by_id(items):
return {item["route_id"]: item for item in items}
before_map = by_id(before)
after_map = by_id(after)
assert "Route Health Admin" in page
assert before_map[primary_route]["runtime_status"] == "cooldown"
assert before_map[failing_route]["runtime_status"] == "failing"
assert resolve["route_id"] == fallback_route
assert resolve["fallback_used"] is True
assert after_map[primary_route]["runtime_status"] == "cooldown"
assert after_map[fallback_route]["runtime_status"] == "healthy"
assert after_map[fallback_route]["recent_failover_count"] >= 1
assert failovers and failovers[0]["from_route_id"] == primary_route
assert failovers[0]["to_route_id"] == fallback_route
assert failovers[0]["reason"] == "active_cooldown:degraded"
summary = {
"group_id": group_id,
"request_id": request_id,
"resolve_route_id": resolve["route_id"],
"resolve_fallback_used": resolve["fallback_used"],
"before_statuses": {k: v["runtime_status"] for k, v in before_map.items()},
"after_statuses": {k: v["runtime_status"] for k, v in after_map.items()},
"fallback_recent_failover_count": after_map[fallback_route]["recent_failover_count"],
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
cat "$ARTIFACT_DIR/12-summary.json"

255
scripts/test/test_real_host_scripts.sh
View File

@@ -660,6 +660,258 @@ EOF
[[ -f "$sensitive_root/20260522_foo/05-subscription-access-prep.sql" ]] || fail "sql file was not moved to sensitive mirror"
}
run_test_verify_route_control_plane_script() {
local tmpdir fakebin artifact_dir stdout_file
tmpdir="$(mktemp -d)"
trap 'rm -rf "$tmpdir"' RETURN
fakebin="$tmpdir/bin"
artifact_dir="$tmpdir/artifacts"
stdout_file="$tmpdir/verify_route_control_plane.stdout.txt"
mkdir -p "$fakebin" "$artifact_dir"
cat > "$fakebin/curl" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
method="GET"
url=""
payload=""
prev=""
for arg in "$@"; do
case "$prev" in
-X) method="$arg"; prev=""; continue ;;
-d|--data) payload="$arg"; prev=""; continue ;;
esac
case "$arg" in
-X|-d|--data) prev="$arg"; continue ;;
http://*|https://*) url="$arg" ;;
esac
done
case "$method $url" in
"POST http://crm.example.com/api/logical-groups")
printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane p2t4-cp-1700000000","status":"active"}}'
;;
"POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/models")
printf '%s\n' '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}'
;;
"POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes")
printf '%s\n' '{"logical_group_route":{"route_id":"primary-1700000000","logical_group_id":"p2t4-cp-1700000000","name":"Primary primary-1700000000","status":"active","priority":10,"weight":100,"shadow_group_id":"shadow-group-1700000000","shadow_host_id":"shadow-host-1700000000"}}'
;;
"POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000/models")
printf '%s\n' '{"logical_group_route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4","status":"active"}}'
;;
"GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000")
printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane p2t4-cp-1700000000","status":"active","models":[{"public_model":"gpt-5.4"}],"routes":[{"route_id":"primary-1700000000"}]}}'
;;
"PUT http://crm.example.com/api/logical-groups/p2t4-cp-1700000000")
printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane Updated p2t4-cp-1700000000","status":"active"}}'
;;
"PUT http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000")
printf '%s\n' '{"logical_group_route":{"route_id":"primary-1700000000","logical_group_id":"p2t4-cp-1700000000","name":"Primary Route Updated","status":"active","priority":12,"weight":80,"shadow_group_id":"shadow-group-1700000000","shadow_host_id":"shadow-host-1700000000"}}'
;;
"GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes")
printf '%s\n' '{"routes":[{"route_id":"primary-1700000000","weight":80}]}'
;;
"GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000/models")
printf '%s\n' '{"models":[{"public_model":"gpt-5.4","shadow_model":"gpt-5.4","status":"active"}]}'
;;
*)
echo "unexpected curl request: $method $url payload=$payload" >&2
exit 1
;;
esac
EOF
chmod +x "$fakebin/curl"
PATH="$fakebin:$PATH" \
CRM_BASE="http://crm.example.com" \
CRM_ADMIN_TOKEN="token" \
TS="1700000000" \
ARTIFACT_DIR="$artifact_dir" \
bash "$ROOT_DIR/scripts/acceptance/verify_route_control_plane.sh" >"$stdout_file"
local summary stdout_text
summary="$(cat "$artifact_dir/10-summary.json")"
stdout_text="$(cat "$stdout_file")"
assert_contains "$summary" '"group_id": "p2t4-cp-1700000000"'
assert_contains "$summary" '"route_id": "primary-1700000000"'
assert_contains "$summary" '"route_updated": true'
assert_contains "$stdout_text" '"route_model_listed": true'
}
run_test_verify_route_data_plane_script() {
local tmpdir fakebin artifact_dir stdout_file payload_log
tmpdir="$(mktemp -d)"
trap 'rm -rf "$tmpdir"' RETURN
fakebin="$tmpdir/bin"
artifact_dir="$tmpdir/artifacts"
stdout_file="$tmpdir/verify_route_data_plane.stdout.txt"
payload_log="$tmpdir/payload.log"
mkdir -p "$fakebin" "$artifact_dir"
cat > "$fakebin/curl" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
method="GET"
url=""
payload=""
prev=""
for arg in "$@"; do
case "$prev" in
-X) method="$arg"; prev=""; continue ;;
-d|--data) payload="$arg"; prev=""; continue ;;
esac
case "$arg" in
-X|-d|--data) prev="$arg"; continue ;;
http://*|https://*) url="$arg" ;;
esac
done
printf '%s\n' "$payload" >> "${PAYLOAD_LOG:?missing PAYLOAD_LOG}"
case "$method $url" in
"POST http://crm.example.com/api/logical-groups")
printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-dp-1700000001","display_name":"P2T4 Data Plane p2t4-dp-1700000001","status":"active"}}'
;;
"POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/models")
printf '%s\n' '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}'
;;
"POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/routes")
printf '%s\n' '{"logical_group_route":{"route_id":"primary-1700000001","shadow_group_id":"shadow-group-9","shadow_host_id":"shadow-host-real"}}'
;;
"POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/routes/primary-1700000001/models")
printf '%s\n' '{"logical_group_route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4"}}'
;;
"POST http://crm.example.com/api/routing/chat/completions")
printf '%s\n' '{"request_id":"req-p2t4-dp-1700000001","logical_group_id":"p2t4-dp-1700000001","model":"gpt-5.4","selected_route":{"route_id":"primary-1700000001","shadow_host_id":"shadow-host-real","shadow_group_id":"shadow-group-9","shadow_model":"gpt-5.4"},"forward":{"upstream_status":200,"effective_gateway_key_source":"managed_subscription"}}'
;;
"GET http://crm.example.com/api/routing/logs/decisions?request_id=req-p2t4-dp-1700000001&limit=5")
printf '%s\n' '{"decision_logs":[{"request_id":"req-p2t4-dp-1700000001","selected_route_id":"primary-1700000001"}]}'
;;
*)
echo "unexpected curl request: $method $url payload=$payload" >&2
exit 1
;;
esac
EOF
chmod +x "$fakebin/curl"
PATH="$fakebin:$PATH" \
PAYLOAD_LOG="$payload_log" \
CRM_BASE="http://crm.example.com" \
CRM_ADMIN_TOKEN="token" \
TS="1700000001" \
SHADOW_HOST_ID="shadow-host-real" \
SHADOW_GROUP_ID="shadow-group-9" \
SUBSCRIPTION_USER_ID="36" \
ARTIFACT_DIR="$artifact_dir" \
bash "$ROOT_DIR/scripts/acceptance/verify_route_data_plane.sh" >"$stdout_file"
local summary payloads
summary="$(cat "$artifact_dir/07-summary.json")"
payloads="$(cat "$payload_log")"
assert_contains "$summary" '"forward_upstream_status": 200'
assert_contains "$summary" '"effective_gateway_key_source": "managed_subscription"'
assert_contains "$payloads" '"subscription_user_id": "36"'
}
run_test_verify_route_health_ui_script() {
local tmpdir fakebin artifact_dir stdout_file
tmpdir="$(mktemp -d)"
trap 'rm -rf "$tmpdir"' RETURN
fakebin="$tmpdir/bin"
artifact_dir="$tmpdir/artifacts"
stdout_file="$tmpdir/verify_route_health_ui.stdout.txt"
mkdir -p "$fakebin" "$artifact_dir"
cat > "$fakebin/curl" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
method="GET"
url=""
payload=""
output_file=""
prev=""
for arg in "$@"; do
case "$prev" in
-X) method="$arg"; prev=""; continue ;;
-d|--data) payload="$arg"; prev=""; continue ;;
-o) output_file="$arg"; prev=""; continue ;;
esac
case "$arg" in
-X|-d|--data|-o) prev="$arg"; continue ;;
http://*|https://*) url="$arg" ;;
esac
done
write_body() {
local body="$1"
if [[ -n "$output_file" ]]; then
printf '%s\n' "$body" > "$output_file"
else
printf '%s\n' "$body"
fi
}
case "$method $url" in
"GET http://portal.example.com/route-health.html")
write_body '<html><title>Route Health Admin</title><body>Route Health Admin</body></html>'
;;
"POST http://crm.example.com/api/logical-groups")
write_body '{"logical_group":{"logical_group_id":"p2t4-health-1700000002","status":"active"}}'
;;
"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/models")
write_body '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}'
;;
"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes")
if [[ "$payload" == *'"route_id":"primary-1700000002"'* ]]; then
write_body '{"logical_group_route":{"route_id":"primary-1700000002"}}'
elif [[ "$payload" == *'"route_id":"fallback-1700000002"'* ]]; then
write_body '{"logical_group_route":{"route_id":"fallback-1700000002"}}'
else
write_body '{"logical_group_route":{"route_id":"failing-1700000002"}}'
fi
;;
"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/primary-1700000002/models"|"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/fallback-1700000002/models"|"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/failing-1700000002/models")
write_body '{"logical_group_route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4"}}'
;;
"POST http://crm.example.com/api/routing/sticky/cooldowns")
write_body '{"route_cooldown":{"route_id":"primary-1700000002","reason":"degraded"}}'
;;
"POST http://crm.example.com/api/routing/sticky/route-failures")
write_body '{"route_failure":{"route_id":"failing-1700000002","failure_count":2,"last_error_class":"timeout"}}'
;;
"GET http://crm.example.com/api/routing/routes/health?logical_group_id=p2t4-health-1700000002")
if [[ ! -f /tmp/p2t4-health-switch ]]; then
write_body '{"route_health":[{"route_id":"primary-1700000002","runtime_status":"cooldown"},{"route_id":"fallback-1700000002","runtime_status":"healthy","recent_failover_count":0},{"route_id":"failing-1700000002","runtime_status":"failing"}]}'
else
write_body '{"route_health":[{"route_id":"primary-1700000002","runtime_status":"cooldown"},{"route_id":"fallback-1700000002","runtime_status":"healthy","recent_failover_count":1,"last_selected_at":"2026-05-29T12:00:00Z"},{"route_id":"failing-1700000002","runtime_status":"failing"}]}'
fi
;;
"POST http://crm.example.com/api/routing/resolve")
: > /tmp/p2t4-health-switch
write_body '{"resolve":{"request_id":"req-p2t4-health-1700000002","route_id":"fallback-1700000002","fallback_used":true}}'
;;
"GET http://crm.example.com/api/routing/logs/failovers?request_id=req-p2t4-health-1700000002&limit=5")
write_body '{"failover_events":[{"from_route_id":"primary-1700000002","to_route_id":"fallback-1700000002","reason":"active_cooldown:degraded"}]}'
;;
*)
echo "unexpected curl request: $method $url payload=$payload" >&2
exit 1
;;
esac
EOF
chmod +x "$fakebin/curl"
PATH="$fakebin:$PATH" \
CRM_BASE="http://crm.example.com" \
CRM_ADMIN_TOKEN="token" \
ROUTE_HEALTH_PAGE_URL="http://portal.example.com/route-health.html" \
TS="1700000002" \
ARTIFACT_DIR="$artifact_dir" \
bash "$ROOT_DIR/scripts/acceptance/verify_route_health_ui.sh" >"$stdout_file"
local summary
summary="$(cat "$artifact_dir/12-summary.json")"
assert_contains "$summary" '"resolve_route_id": "fallback-1700000002"'
assert_contains "$summary" '"fallback_recent_failover_count": 1'
}
run_test_remote43_patched_stack_renderers() {
# shellcheck disable=SC1091
source "$ROOT_DIR/scripts/deploy/remote43_patched_stack_lib.sh"
@@ -784,6 +1036,9 @@ run_test_real_host_acceptance_after_import_hook
run_test_check_deepseek_completion_split
run_test_import_remote43_provider_subscription_prep
run_test_migrate_historical_artifacts
run_test_verify_route_control_plane_script
run_test_verify_route_data_plane_script
run_test_verify_route_health_ui_script
run_test_remote43_patched_stack_renderers
run_test_setup_remote43_patched_stack_dry_run