492f33a129
- DEFAULT_CHAIN_ADMISSION.md: reviewed and approved, real artifact refs added - DEFAULT_DATA_IDEMPOTENT_RELEASE_GATE.md: reviewed and approved - scripts/setup_default_data.sh: idempotent init with --dry-run/--apply/artifact - scripts/test/test_default_data.sh: 4 test cases all pass - scripts/acceptance/verify_user_key_self_service.sh: Phase 0 skeleton - .gitignore: add generated artifact directories
202 lines
6.8 KiB
Bash
202 lines
6.8 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
SCRIPT="$ROOT_DIR/scripts/acceptance/verify_host_protocol_matrix.sh"
|
|
|
|
fail() {
|
|
echo "FAIL: $*" >&2
|
|
exit 1
|
|
}
|
|
|
|
assert_contains() {
|
|
local haystack="$1"
|
|
local needle="$2"
|
|
if [[ "$haystack" != *"$needle"* ]]; then
|
|
fail "expected to find [$needle] in [$haystack]"
|
|
fi
|
|
}
|
|
|
|
assert_file_contains() {
|
|
local file="$1"
|
|
local needle="$2"
|
|
[[ -f "$file" ]] || fail "missing file: $file"
|
|
local text
|
|
text="$(cat "$file")"
|
|
assert_contains "$text" "$needle"
|
|
}
|
|
|
|
[[ -f "$SCRIPT" ]] || fail "missing $SCRIPT"
|
|
|
|
help_output="$(bash "$SCRIPT" --help)"
|
|
assert_contains "$help_output" "Usage: verify_host_protocol_matrix.sh"
|
|
assert_contains "$help_output" "PROTOCOL_MATRIX_TARGETS_JSON"
|
|
assert_contains "$help_output" "DRY_RUN=1"
|
|
|
|
tmpdir="$(mktemp -d)"
|
|
trap 'rm -rf "$tmpdir"' EXIT
|
|
|
|
set +e
|
|
missing_env_output="$(ARTIFACT_DIR="$tmpdir" bash "$SCRIPT" 2>&1)"
|
|
missing_env_status=$?
|
|
set -e
|
|
if [[ $missing_env_status -eq 0 ]]; then
|
|
fail "expected missing env invocation to fail"
|
|
fi
|
|
assert_contains "$missing_env_output" "missing required env: PROTOCOL_MATRIX_TARGETS_JSON"
|
|
|
|
dry_run_output="$(ARTIFACT_DIR="$tmpdir" DRY_RUN=1 PROTOCOL_MATRIX_TARGETS_JSON='[{"provider_id":"kimi-a7m","base_url":"https://kimi.example.com/v1","api_key_env":"KIMI_API_KEY","models":["kimi-k2.6"]}]' bash "$SCRIPT")"
|
|
assert_contains "$dry_run_output" "protocol matrix summary"
|
|
assert_contains "$dry_run_output" "$tmpdir"
|
|
|
|
summary_file="$(find "$tmpdir" -name protocol-matrix-summary.json | head -n 1)"
|
|
[[ -n "$summary_file" ]] || fail "missing protocol-matrix-summary.json"
|
|
summary_text="$(cat "$summary_file")"
|
|
assert_contains "$summary_text" '"provider_id": "kimi-a7m"'
|
|
assert_contains "$summary_text" '"mode": "dry_run"'
|
|
assert_contains "$summary_text" '"support_level": "dry_run"'
|
|
assert_contains "$summary_text" '"probe_layer": "upstream"'
|
|
|
|
fakebin="$tmpdir/bin"
|
|
mkdir -p "$fakebin"
|
|
cat > "$fakebin/curl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
headers_file=""
|
|
body_file=""
|
|
url=""
|
|
request_headers_file=""
|
|
request_body=""
|
|
prev=""
|
|
log_file="${FAKE_CURL_LOG:-}"
|
|
for arg in "$@"; do
|
|
case "$prev" in
|
|
-D)
|
|
headers_file="$arg"
|
|
prev=""
|
|
continue
|
|
;;
|
|
-o)
|
|
body_file="$arg"
|
|
prev=""
|
|
continue
|
|
;;
|
|
-d)
|
|
request_body="$arg"
|
|
prev=""
|
|
continue
|
|
;;
|
|
-H)
|
|
if [[ "$arg" == X-Hermes-Debug-Request-Headers:* ]]; then
|
|
request_headers_file="${arg#X-Hermes-Debug-Request-Headers: }"
|
|
fi
|
|
prev=""
|
|
continue
|
|
;;
|
|
esac
|
|
case "$arg" in
|
|
-D|-o|-d|-H)
|
|
prev="$arg"
|
|
continue
|
|
;;
|
|
http://*|https://*)
|
|
url="$arg"
|
|
;;
|
|
esac
|
|
done
|
|
[[ -n "$headers_file" && -n "$body_file" && -n "$url" ]] || {
|
|
echo "missing curl capture args: $*" >&2
|
|
exit 1
|
|
}
|
|
if [[ -n "$log_file" ]]; then
|
|
printf '%s\n' "$*" >> "$log_file"
|
|
fi
|
|
if [[ -n "$request_headers_file" ]]; then
|
|
printf 'Authorization: Bearer ***\n' > "$request_headers_file"
|
|
printf 'Content-Type: application/json\n' >> "$request_headers_file"
|
|
fi
|
|
case "$url" in
|
|
https://kimi.example.com/v1/models)
|
|
printf 'HTTP/1.1 200 OK\nContent-Type: application/json\n' > "$headers_file"
|
|
printf '{"data":[{"id":"kimi-k2.6"}]}' > "$body_file"
|
|
;;
|
|
https://kimi.example.com/v1/chat/completions)
|
|
printf 'HTTP/1.1 200 OK\nContent-Type: application/json\n' > "$headers_file"
|
|
printf '{"choices":[{"message":{"content":"pong"}}]}' > "$body_file"
|
|
;;
|
|
https://kimi.example.com/v1/responses)
|
|
printf 'HTTP/1.1 403 Forbidden\nContent-Type: application/json\n' > "$headers_file"
|
|
printf '{"error":{"message":"unsupported"}}' > "$body_file"
|
|
;;
|
|
https://timeout.example.com/v1/models)
|
|
printf 'HTTP/1.1 200 OK\nContent-Type: application/json\n' > "$headers_file"
|
|
printf '{"data":[{"id":"timeout-model"}]}' > "$body_file"
|
|
;;
|
|
https://timeout.example.com/v1/chat/completions)
|
|
: > "$headers_file"
|
|
: > "$body_file"
|
|
exit 28
|
|
;;
|
|
https://timeout.example.com/v1/responses)
|
|
: > "$headers_file"
|
|
: > "$body_file"
|
|
exit 28
|
|
;;
|
|
*)
|
|
echo "unexpected curl url: $url" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
EOF
|
|
chmod +x "$fakebin/curl"
|
|
|
|
live_dir="$tmpdir/live"
|
|
curl_log="$tmpdir/fake-curl.log"
|
|
set +e
|
|
live_output="$(PATH="$fakebin:$PATH" FAKE_CURL_LOG="$curl_log" ARTIFACT_DIR="$live_dir" KIMI_API_KEY='kimi-key' TIMEOUT_API_KEY='timeout-key' PROTOCOL_MATRIX_TARGETS_JSON='[
|
|
{"provider_id":"kimi-a7m","base_url":"https://kimi.example.com/v1","api_key_env":"KIMI_API_KEY","models":["kimi-k2.6"]},
|
|
{"provider_id":"timeout-provider","base_url":"https://timeout.example.com/v1","api_key_env":"TIMEOUT_API_KEY","models":["timeout-model"],"probe_layer":"host"}
|
|
]' bash "$SCRIPT")"
|
|
live_status=$?
|
|
set -e
|
|
if [[ $live_status -ne 0 ]]; then
|
|
fail "expected partial failure run to exit 0, got $live_status"
|
|
fi
|
|
assert_contains "$live_output" "protocol matrix summary"
|
|
live_summary="$live_dir/protocol-matrix-summary.json"
|
|
[[ -f "$live_summary" ]] || fail "missing live protocol-matrix-summary.json"
|
|
live_summary_text="$(cat "$live_summary")"
|
|
assert_contains "$live_summary_text" '"provider_id": "kimi-a7m"'
|
|
assert_contains "$live_summary_text" '"models_status": 200'
|
|
assert_contains "$live_summary_text" '"chat_status": 200'
|
|
assert_contains "$live_summary_text" '"responses_status": 403'
|
|
assert_contains "$live_summary_text" '"support_level": "supported-with-plugin-adapter"'
|
|
assert_contains "$live_summary_text" '"error_code": "responses_unsupported"'
|
|
assert_contains "$live_summary_text" '"probe_layer": "upstream"'
|
|
assert_contains "$live_summary_text" '"provider_id": "timeout-provider"'
|
|
assert_contains "$live_summary_text" '"status": "failed"'
|
|
assert_contains "$live_summary_text" '"error_code": "network_timeout"'
|
|
assert_contains "$live_summary_text" '"probe_layer": "host"'
|
|
|
|
first_target_dir="$live_dir/targets/01-kimi-a7m"
|
|
second_target_dir="$live_dir/targets/02-timeout-provider"
|
|
[[ -d "$first_target_dir" ]] || fail "missing first target artifact dir"
|
|
[[ -d "$second_target_dir" ]] || fail "missing second target artifact dir"
|
|
assert_file_contains "$first_target_dir/01-models.request_headers.txt" 'Authorization: Bearer ***'
|
|
assert_file_contains "$first_target_dir/01-models.request_headers.txt" 'Content-Type: application/json'
|
|
assert_file_contains "$first_target_dir/01-models.response_headers.txt" 'HTTP/1.1 200 OK'
|
|
assert_file_contains "$first_target_dir/03-responses.response_body.json" 'unsupported'
|
|
assert_file_contains "$second_target_dir/02-chat.response_body.json" ''
|
|
|
|
curl_log_text="$(cat "$curl_log")"
|
|
assert_contains "$curl_log_text" '--connect-timeout 10'
|
|
assert_contains "$curl_log_text" '--max-time 30'
|
|
assert_contains "$curl_log_text" '--retry 1'
|
|
assert_contains "$curl_log_text" '--retry-delay 2'
|
|
|
|
if grep -R --line-number 'kimi-key\|timeout-key' "$live_dir"; then
|
|
fail "artifact contains unsanitized secrets"
|
|
fi
|
|
|
|
echo "PASS: host protocol matrix script regression checks"
|