161 lines
5.8 KiB
Go
161 lines
5.8 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
func TestAPIListRouteHealthRejectsInvalidStatus(t *testing.T) {
|
|
handler := NewAPIHandler("secret-token", ActionSet{
|
|
ListRouteHealth: func(context.Context, ListRouteHealthRequest) ([]RouteHealthInfo, error) {
|
|
t.Fatal("ListRouteHealth should not be called")
|
|
return nil, nil
|
|
},
|
|
})
|
|
|
|
request := httptestRequest(t, http.MethodGet, "/api/routing/routes/health?status=broken", nil, "secret-token")
|
|
response := httptestRecorder(handler, request)
|
|
assertStatusCode(t, response, http.StatusBadRequest)
|
|
assertJSONContains(t, response.Body().Bytes(), "error.code", "bad_request")
|
|
}
|
|
|
|
func TestNewActionSetRouteHealthFlow(t *testing.T) {
|
|
dsn := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "route-health.db")) + "?_busy_timeout=5000"
|
|
actions := NewActionSet(dsn)
|
|
ctx := context.Background()
|
|
|
|
if _, err := actions.CreateLogicalGroup(ctx, CreateLogicalGroupRequest{
|
|
LogicalGroupID: "gpt-shared",
|
|
DisplayName: "GPT Shared",
|
|
Status: "active",
|
|
RoutePolicy: "priority",
|
|
StickyMode: "conversation_preferred",
|
|
ConversationTTLSeconds: 1200,
|
|
UserModelTTLSeconds: 600,
|
|
FailoverThreshold: 2,
|
|
CooldownSeconds: 300,
|
|
}); err != nil {
|
|
t.Fatalf("CreateLogicalGroup(gpt-shared) error = %v", err)
|
|
}
|
|
|
|
createRoute := func(routeID, name, status, shadowGroup string, priority int) {
|
|
t.Helper()
|
|
if _, err := actions.CreateLogicalGroupRoute(ctx, CreateLogicalGroupRouteRequest{
|
|
LogicalGroupID: "gpt-shared",
|
|
RouteID: routeID,
|
|
Name: name,
|
|
Status: status,
|
|
Priority: priority,
|
|
ShadowGroupID: shadowGroup,
|
|
ShadowHostID: "remote43",
|
|
UpstreamBaseURLHint: "https://example.com/v1",
|
|
}); err != nil {
|
|
t.Fatalf("CreateLogicalGroupRoute(%s) error = %v", routeID, err)
|
|
}
|
|
}
|
|
|
|
createRoute("healthy-route", "Healthy", "active", "shadow-healthy", 10)
|
|
createRoute("failing-route", "Failing", "active", "shadow-failing", 20)
|
|
createRoute("cooldown-route", "Cooldown", "active", "shadow-cooldown", 30)
|
|
createRoute("disabled-route", "Disabled", "disabled", "shadow-disabled", 40)
|
|
|
|
if _, err := actions.AppendRouteDecisionLog(ctx, AppendRouteDecisionLogRequest{
|
|
RequestID: "req-healthy",
|
|
LogicalGroupID: "gpt-shared",
|
|
PublicModel: "gpt-5.4",
|
|
SelectedRouteID: "healthy-route",
|
|
SelectedShadowGroupID: "shadow-healthy",
|
|
UpstreamStatus: 200,
|
|
Sync: true,
|
|
}); err != nil {
|
|
t.Fatalf("AppendRouteDecisionLog(healthy) error = %v", err)
|
|
}
|
|
|
|
if _, err := actions.AppendRouteDecisionLog(ctx, AppendRouteDecisionLogRequest{
|
|
RequestID: "req-failing",
|
|
LogicalGroupID: "gpt-shared",
|
|
PublicModel: "gpt-5.4",
|
|
SelectedRouteID: "failing-route",
|
|
SelectedShadowGroupID: "shadow-failing",
|
|
ErrorClass: "upstream_5xx",
|
|
UpstreamStatus: 503,
|
|
Sync: true,
|
|
}); err != nil {
|
|
t.Fatalf("AppendRouteDecisionLog(failing) error = %v", err)
|
|
}
|
|
|
|
if _, err := actions.AppendRouteFailoverEvent(ctx, AppendRouteFailoverEventRequest{
|
|
RequestID: "req-failover",
|
|
LogicalGroupID: "gpt-shared",
|
|
PublicModel: "gpt-5.4",
|
|
FromRouteID: "failing-route",
|
|
ToRouteID: "healthy-route",
|
|
Reason: "failure_threshold_exceeded:timeout",
|
|
FailureCount: 2,
|
|
Sync: true,
|
|
}); err != nil {
|
|
t.Fatalf("AppendRouteFailoverEvent() error = %v", err)
|
|
}
|
|
|
|
if _, err := actions.SetRouteFailure(ctx, SetRouteFailureRequest{
|
|
RouteID: "failing-route",
|
|
FailureCount: 2,
|
|
LastErrorClass: "timeout",
|
|
TTLSeconds: 600,
|
|
}); err != nil {
|
|
t.Fatalf("SetRouteFailure() error = %v", err)
|
|
}
|
|
|
|
if _, err := actions.SetRouteCooldown(ctx, SetRouteCooldownRequest{
|
|
RouteID: "cooldown-route",
|
|
Reason: "degraded",
|
|
TTLSeconds: 600,
|
|
}); err != nil {
|
|
t.Fatalf("SetRouteCooldown() error = %v", err)
|
|
}
|
|
|
|
items, err := actions.ListRouteHealth(ctx, ListRouteHealthRequest{})
|
|
if err != nil {
|
|
t.Fatalf("ListRouteHealth() error = %v", err)
|
|
}
|
|
if len(items) != 4 {
|
|
t.Fatalf("ListRouteHealth() len = %d, want 4", len(items))
|
|
}
|
|
|
|
byRoute := make(map[string]RouteHealthInfo, len(items))
|
|
for _, item := range items {
|
|
byRoute[item.RouteID] = item
|
|
}
|
|
|
|
if got := byRoute["healthy-route"]; got.RuntimeStatus != routeRuntimeStatusHealthy || got.LastUpstreamStatus != 200 {
|
|
t.Fatalf("healthy-route = %+v, want healthy with upstream 200", got)
|
|
}
|
|
if got := byRoute["failing-route"]; got.RuntimeStatus != routeRuntimeStatusFailing || got.FailureCount != 2 || got.LastErrorClass != "timeout" || got.RecentFailoverCount != 1 || got.LastUpstreamStatus != 503 {
|
|
t.Fatalf("failing-route = %+v, want failing with failure_count=2 recent_failover_count=1 upstream=503", got)
|
|
}
|
|
if got := byRoute["cooldown-route"]; got.RuntimeStatus != routeRuntimeStatusCooldown || got.CooldownReason != "degraded" || got.CooldownUntil == "" {
|
|
t.Fatalf("cooldown-route = %+v, want cooldown with reason degraded", got)
|
|
}
|
|
if got := byRoute["disabled-route"]; got.RuntimeStatus != routeRuntimeStatusDisabled {
|
|
t.Fatalf("disabled-route = %+v, want disabled", got)
|
|
}
|
|
|
|
failingOnly, err := actions.ListRouteHealth(ctx, ListRouteHealthRequest{Status: routeRuntimeStatusFailing})
|
|
if err != nil {
|
|
t.Fatalf("ListRouteHealth(failing) error = %v", err)
|
|
}
|
|
if len(failingOnly) != 1 || failingOnly[0].RouteID != "failing-route" {
|
|
t.Fatalf("ListRouteHealth(failing) = %+v, want only failing-route", failingOnly)
|
|
}
|
|
|
|
oneRoute, err := actions.ListRouteHealth(ctx, ListRouteHealthRequest{RouteID: "cooldown-route"})
|
|
if err != nil {
|
|
t.Fatalf("ListRouteHealth(route_id) error = %v", err)
|
|
}
|
|
if len(oneRoute) != 1 || oneRoute[0].RouteID != "cooldown-route" {
|
|
t.Fatalf("ListRouteHealth(route_id) = %+v, want only cooldown-route", oneRoute)
|
|
}
|
|
}
|