Complete batch import v2 runtime and host capability recovery
This commit is contained in:
@@ -13,6 +13,7 @@ import (
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
"sub2api-cn-relay-manager/internal/pack"
|
||||
"sub2api-cn-relay-manager/internal/provision"
|
||||
"sub2api-cn-relay-manager/internal/reconcile"
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
)
|
||||
|
||||
@@ -20,7 +21,7 @@ type installPackFunc func(context.Context, installPackCLIRequest) (provision.Pac
|
||||
type importProviderFunc func(context.Context, importCLIRequest) (provision.ImportReport, error)
|
||||
type previewProviderFunc func(context.Context, previewCLIRequest) (provision.PreviewReport, error)
|
||||
type rollbackProviderFunc func(context.Context, rollbackCLIRequest) (rollbackSummary, error)
|
||||
type reconcileProviderFunc func(context.Context, reconcileCLIRequest) (provision.ReconcileResult, error)
|
||||
type reconcileProviderFunc func(context.Context, reconcileCLIRequest) (reconcile.Result, error)
|
||||
|
||||
type installPackCLIRequest struct {
|
||||
HostBaseURL string
|
||||
@@ -445,34 +446,39 @@ func runRollbackProvider(ctx context.Context, req rollbackCLIRequest) (rollbackS
|
||||
}, nil
|
||||
}
|
||||
|
||||
func runReconcileProvider(ctx context.Context, req reconcileCLIRequest) (provision.ReconcileResult, error) {
|
||||
func runReconcileProvider(ctx context.Context, req reconcileCLIRequest) (reconcile.Result, error) {
|
||||
loadedPack, err := pack.LoadDir(req.PackDir)
|
||||
if err != nil {
|
||||
return provision.ReconcileResult{}, err
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
providerManifest, err := findProvider(loadedPack, req.ProviderID)
|
||||
if err != nil {
|
||||
return provision.ReconcileResult{}, err
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
client, err := sub2api.NewClient(req.HostBaseURL, sub2api.WithAPIKey(req.HostAPIKey), sub2api.WithBearerToken(req.HostBearerToken))
|
||||
if err != nil {
|
||||
return provision.ReconcileResult{}, err
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
startupConfig, err := config.LoadStartupFromEnv()
|
||||
if err != nil {
|
||||
return provision.ReconcileResult{}, err
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
store, err := sqlite.Open(ctx, startupConfig.Database.SQLiteDSN)
|
||||
if err != nil {
|
||||
return provision.ReconcileResult{}, err
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
service := provision.NewReconcileService(store, client)
|
||||
return service.Reconcile(ctx, provision.ReconcileRequest{HostBaseURL: req.HostBaseURL, AccessProbeAPIKey: req.AccessAPIKey, Pack: loadedPack, Provider: providerManifest})
|
||||
hostRow, err := store.Hosts().GetByBaseURL(ctx, req.HostBaseURL)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
service := reconcile.NewService(store, client)
|
||||
return service.Reconcile(ctx, reconcile.Request{HostID: hostRow.HostID, HostBaseURL: req.HostBaseURL, AccessProbeAPIKey: req.AccessAPIKey, Pack: loadedPack, Provider: providerManifest})
|
||||
}
|
||||
|
||||
func findProvider(loaded pack.LoadedPack, providerID string) (pack.ProviderManifest, error) {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"sub2api-cn-relay-manager/internal/config"
|
||||
"sub2api-cn-relay-manager/internal/provision"
|
||||
"sub2api-cn-relay-manager/internal/reconcile"
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
)
|
||||
|
||||
@@ -221,12 +222,12 @@ func TestExecuteReconcileProviderWritesSummary(t *testing.T) {
|
||||
"--pack-dir", "/tmp/pack",
|
||||
"--provider-id", "deepseek",
|
||||
"--access-api-key", "user-key",
|
||||
}, nil, nil, nil, nil, nil, func(_ context.Context, req reconcileCLIRequest) (provision.ReconcileResult, error) {
|
||||
}, nil, nil, nil, nil, nil, func(_ context.Context, req reconcileCLIRequest) (reconcile.Result, error) {
|
||||
reconcileCalled = true
|
||||
if req.ProviderID != "deepseek" || req.AccessAPIKey != "user-key" {
|
||||
t.Fatalf("unexpected reconcile request: %+v", req)
|
||||
}
|
||||
return provision.ReconcileResult{Status: "drifted", MissingCount: 1, ExtraCount: 2, ProbeFailureCount: 1, AccessStatus: provision.AccessStatusBroken}, nil
|
||||
return reconcile.Result{Status: "drifted", MissingCount: 1, ExtraCount: 2, ProbeFailureCount: 1, AccessStatus: provision.AccessStatusBroken}, nil
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("execute() reconcile error = %v", err)
|
||||
|
||||
@@ -58,7 +58,8 @@
|
||||
- 能力探测:[capability_probe.go](/home/long/project/sub2api-cn-relay-manager/internal/host/sub2api/capability_probe.go:1)
|
||||
- 导入运行时:[runtime_import_service.go](/home/long/project/sub2api-cn-relay-manager/internal/provision/runtime_import_service.go:1)
|
||||
- 回滚:[rollback_service.go](/home/long/project/sub2api-cn-relay-manager/internal/provision/rollback_service.go:1)
|
||||
- 对账:[batch_detail_and_reconcile_service.go](/home/long/project/sub2api-cn-relay-manager/internal/provision/batch_detail_and_reconcile_service.go:1)
|
||||
- 对账:[service.go](/home/long/project/sub2api-cn-relay-manager/internal/reconcile/service.go:1)
|
||||
- batch detail:[batch_detail_service.go](/home/long/project/sub2api-cn-relay-manager/internal/provision/batch_detail_service.go:1)
|
||||
- 状态库:[db.go](/home/long/project/sub2api-cn-relay-manager/internal/store/sqlite/db.go:1)
|
||||
- 资源记录:[managed_resources_repo.go](/home/long/project/sub2api-cn-relay-manager/internal/store/sqlite/managed_resources_repo.go:1)
|
||||
|
||||
@@ -217,8 +218,8 @@
|
||||
|
||||
证据:
|
||||
|
||||
- implementation plan 里期望的 `internal/reconcile/*`、`access/planner.go`、`worker/scheduler.go` 等结构仍未落地,[implementation-plan.md](/home/long/project/sub2api-cn-relay-manager/docs/plans/2026-05-12-sub2api-cn-relay-manager-implementation-plan.md:69)
|
||||
- 当前逻辑主要仍集中在 `internal/provision/*` 与 `internal/access/closure.go`。
|
||||
- 该评审形成时,implementation plan 里期望的 `internal/reconcile/*`、`access/planner.go`、`worker/scheduler.go` 等结构仍未落地,[implementation-plan.md](/home/long/project/sub2api-cn-relay-manager/docs/plans/2026-05-12-sub2api-cn-relay-manager-implementation-plan.md:69)
|
||||
- 截至 2026-05-22,这些结构项已分别落到 `internal/reconcile/*`、`internal/access/{planner,subscription,self_service,validation}.go` 与 `internal/worker/*`。
|
||||
|
||||
影响:
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
|
||||
- [x] 单一状态源为 `import_runs / import_run_items / import_run_item_events`
|
||||
- [x] migration 已落地并受集成测试保护
|
||||
- [x] run 级请求上下文(`host_id / subscription_users / subscription_days / probe_api_key`)已持久化,支持重启后恢复 validate
|
||||
- [x] `/api/batch-import/runs*` 已接到 V2 projection
|
||||
- [x] CLI `batch-import` 已通过 `ActionSet.CreateBatchImportRun` 进入真实 pipeline
|
||||
- [x] 结果页/结果 API 不回退 legacy 表结构
|
||||
@@ -54,14 +55,14 @@
|
||||
|
||||
- Probe / alias / capability:`internal/probe/models.go`、`internal/probe/aliases.go`、`internal/probe/capability.go`、`internal/probe/completion.go`
|
||||
- Reuse / orchestration / worker / validation:`internal/batch/provider_id.go`、`internal/batch/reuse_policy.go`、`internal/batch/service.go`、`internal/batch/confirmation.go`、`internal/batch/validation.go`
|
||||
- 状态库存储:`internal/store/sqlite/import_runs_repo.go`、`internal/store/sqlite/import_run_items_repo.go`、`internal/store/sqlite/import_run_item_events_repo.go`
|
||||
- Projection / API / CLI:`internal/batch/status_projection.go`、`internal/app/http_batch_import.go`、`internal/app/batch_runtime.go`、`internal/app/http_batch_runs.go`、`cmd/cli/batch_import.go`
|
||||
- 状态库存储:`internal/store/sqlite/import_runs_repo.go`、`internal/store/sqlite/import_run_items_repo.go`、`internal/store/sqlite/import_run_item_events_repo.go`、`internal/store/migrations/0009_batch_import_run_request_context.sql`
|
||||
- Projection / API / CLI:`internal/batch/status_projection.go`、`internal/app/http_batch_import.go`、`internal/app/batch_runtime.go`、`internal/app/batch_runtime_background.go`、`internal/app/http_batch_runs.go`、`cmd/cli/batch_import.go`
|
||||
|
||||
### 测试文件映射
|
||||
|
||||
- 单测:`internal/batch/types_test.go`、`internal/probe/models_test.go`、`internal/probe/aliases_test.go`、`internal/probe/capability_test.go`、`internal/probe/completion_test.go`
|
||||
- 状态机:`internal/batch/provider_id_test.go`、`internal/batch/reuse_policy_test.go`、`internal/batch/service_test.go`、`internal/batch/confirmation_test.go`、`internal/batch/validation_test.go`、`internal/batch/status_projection_test.go`
|
||||
- API / CLI:`internal/app/http_batch_import_test.go`、`internal/app/http_batch_runs_test.go`、`cmd/cli/batch_import_test.go`
|
||||
- API / CLI:`internal/app/http_batch_import_test.go`、`internal/app/http_batch_runs_test.go`、`internal/app/batch_runtime_background_test.go`、`cmd/cli/batch_import_test.go`
|
||||
- 集成:`tests/integration/batch_import_v2_test.go`
|
||||
|
||||
### API 路由映射
|
||||
@@ -78,7 +79,7 @@
|
||||
- `go test ./tests/integration/... -count=1`:PASS
|
||||
- `go test -cover ./internal/... -count=1`:PASS
|
||||
- `internal/access` 76.7%
|
||||
- `internal/batch` 75.4%
|
||||
- `internal/batch` 72.9%
|
||||
- `internal/probe` 78.2%
|
||||
- `internal/provision` 76.4%
|
||||
- `internal/pack` 73.9%
|
||||
@@ -89,6 +90,9 @@
|
||||
|
||||
- `internal/app/http_batch_import.go` 的 `buildCreateBatchImportRunAction` 已改为先解析已注册 host,再委托 `batchImportRuntimeRunner.execute`
|
||||
- `internal/app/batch_runtime.go` 已把 `BatchImportService + ConfirmationWorker + ValidationService` 串成 create-run 的同步入口驱动链
|
||||
- `internal/app/batch_runtime_background.go` 已补后台 runtime scheduler;`running` run 在控制面重启后会继续被拾取并推进
|
||||
- `internal/store/sqlite/import_run_items_repo.go` 已补原子 lease 获取;不会再在 lease 落库前并发双发 confirmer
|
||||
- `internal/app/http_batch_import.go` / `internal/app/http_batch_runs.go` 已补 `cursor/next_cursor`,且 run 列表 `q` 可命中 `run_id / provider_id / base_url`
|
||||
- `cmd/cli/batch_import.go` 继续复用 `ActionSet.CreateBatchImportRun`,因此 CLI create-run 也随入口修复自动进入真实 pipeline
|
||||
- `internal/app/http_batch_import_test.go` 已新增真实 stub 回归,直接验证 create-run 最终把 item 推进到 `current_stage=done` 且 `access_status=active`
|
||||
|
||||
|
||||
@@ -15,6 +15,9 @@
|
||||
- `self_service` 主链路已通过 latest-head 标准 fresh-host 复验:
|
||||
- `artifacts/real-host-acceptance/20260521_210403/05-import.json`
|
||||
- `artifacts/real-host-acceptance/20260521_210403/07-access-status.json`
|
||||
- latest-head relay-manager 已新增宿主 capability 自愈:
|
||||
- 当第三方 OpenAI-compatible upstream 因宿主把 `openai_responses_supported` 误判成 `true` 而导致 host `/v1/chat/completions` 返回 `502 upstream_error` 时,access closure 与后台 reconcile 会自动把相关 account 修正到 raw `/chat/completions` 路径后再重试
|
||||
- 该修正现在不再依赖宿主长期保留补丁,宿主升级后只要下次 import/access/reconcile 触发,就能重新收敛到正确 capability
|
||||
- 官方 provider 验证矩阵当前仍保留一条非阻塞事实:
|
||||
- `artifacts/real-host-acceptance/20260521_222212_remote43_minimax-m2-7-official_key_import/21-summary.json` 已证明 official MiniMax 模板链路是通的,但该验证 key 当前命中 upstream `429`
|
||||
- `reconcile=drifted` 仍可能在 shared fresh-host 上出现,但当前解释是“历史残留资源噪音”,不阻塞 PRD 首版放行
|
||||
@@ -69,6 +72,9 @@
|
||||
- account test 首次 `403 Forbidden` 已降级为 advisory warning;只要 `/models` 已命中 `smoke_test_model`,不会再把 batch 误判为 blocking failure
|
||||
- access closure 对导入后瞬时 `503 / no available accounts` 增加短暂 completion retry,避免宿主异步 probe / account warm-up 窗口把真实可用链路误记成 `broken`
|
||||
- `20260522_122706_local_v0129_kimi_a7m_subscription_freshhost` 已证明:在修复后的 relay-manager + patched host 组合下,`kimi-a7m / kimi-k2.6` 可落到 `batch_status=succeeded`、`provider_status=active`、`latest_access_status=subscription_ready`
|
||||
14. relay-manager latest-head 已补宿主升级后的 capability 自愈
|
||||
- 对 `API returned 403: Forbidden` 这类 `/responses` 误判 advisory,控制面现在会在 access closure 与 reconcile rerun 中把目标 account 的 `openai_responses_supported` 修正为 `false`,随后重试 gateway `/v1/chat/completions`
|
||||
- 这样即使宿主升级或异步 probe 把 capability 标记覆写错,控制面也能在“安装后确认”与“后台持续对账”两个环节重新拉回可用状态
|
||||
|
||||
## 已验证门禁
|
||||
|
||||
@@ -122,16 +128,13 @@
|
||||
- 真实宿主初始化不会自动创建普通用户;上线前必须显式创建普通用户并留存可复用凭据
|
||||
- `self_service` 需要普通用户 key 绑定目标标准 group,且通常还需要可用余额
|
||||
- `subscription` 需要 subscription 类型 group + 普通用户订阅分配 + key/group 绑定
|
||||
- 若启用持续后台 reconcile,SQLite 状态库将持久化最新 access probe 元数据,部署时必须按 secret 级别保护数据库文件
|
||||
|
||||
2. 结构债务
|
||||
- access / reconcile 仍未完全按 implementation plan 拆到独立子模块
|
||||
- 当前仍无内置 scheduler/jobs
|
||||
|
||||
3. 部署与环境限制
|
||||
2. 部署与环境限制
|
||||
- 标准多阶段 Dockerfile 在受限网络环境下仍不稳
|
||||
- 当前推荐 `scripts/build_local_image.sh` + `Dockerfile.local`
|
||||
|
||||
4. official provider 验证矩阵
|
||||
3. official provider 验证矩阵
|
||||
- official MiniMax 当前 live 样本已证明模板链路可用,但验证 key 命中 upstream `429`
|
||||
- Qwen / GLM / Kimi / Step 等官方 provider 是否通过 live 验收,仍取决于后续官方 key 与 quota
|
||||
|
||||
@@ -168,7 +171,9 @@
|
||||
- 其余 review 问题也已同步收口:
|
||||
- capability 从 upstream 总画像升级为 transport + model profiles
|
||||
- 结果页字段、状态库存储字段、retry/event trail 已统一
|
||||
- run 级请求上下文已持久化到 `import_runs`,控制面重启后 validate 能继续使用 `host_id / subscription_users / subscription_days / probe_api_key`
|
||||
- OpenAPI 已补齐 `/api/batch-import/runs*`,legacy `/api/import-batches/*` 降级为 v1/legacy
|
||||
- run/item 列表 API 已补齐 `cursor/next_cursor`;run 列表 `q` 可命中 `run_id / provider_id / base_url`
|
||||
- 已补充重复导入自动复用策略:按 `provider_id + api_key_fingerprint + canonical_model_family` 判断 `reused / patch_only / replace`
|
||||
- 已补充同模型别名归一化契约:例如 `kimi 2.6 / kimi-2.6 / kimi-k2.6` 可归并到同一模型家族并快速复用
|
||||
- 已补充多账号重复导入与弃用账号再启用策略:active 账号提示“重复已启用”,disabled/deprecated 账号显示原状态并走 `reactivated` 快速启用路径
|
||||
@@ -188,6 +193,7 @@
|
||||
- `docs/2026-05-22-BATCH_AUTO_IMPORT_V2_RESTORATION_CHECKLIST.md` 已完成
|
||||
- latest-head 已补齐 `internal/app/http_batch_import.go` -> `internal/app/batch_runtime.go` 的 create-run 入口 wiring
|
||||
- API 与 CLI create-run 现在都会真实驱动 `BatchImportService + ConfirmationWorker + ValidationService`
|
||||
- 控制面 server 启动后会自动运行 batch-import background scheduler,`running` run 在重启后可继续推进
|
||||
- 最新一轮验证结果保持全绿:`go test ./... -count=1`、`go test ./tests/integration/... -count=1`、`go test -cover ./internal/... -count=1`、`go vet ./...`、`gofmt -l .`
|
||||
|
||||
**真实 Gate**:✅ 文档、状态机、投影、测试、审计与 create-run 入口已经对齐,**V2 设计已按基线计划交付**
|
||||
|
||||
@@ -4,10 +4,11 @@ This document covers known limitations that operators should be aware of before
|
||||
|
||||
## Core Limitations
|
||||
|
||||
### 1. No Automated Reconcile Scheduler (P2)
|
||||
- Reconcilation must be triggered manually via `POST /api/providers/{providerID}/reconcile` or CLI.
|
||||
- No cron/scheduler service is bundled.
|
||||
- Workaround: set up a cron job on the host OS calling the HTTP API periodically.
|
||||
### 1. Automated Reconcile Is Available, but Disabled by Default (P2)
|
||||
- A built-in background reconcile worker is now available in the control plane server.
|
||||
- It is gated by `SUB2API_CRM_RECONCILE_WORKER_ENABLED=true` and uses `SUB2API_CRM_RECONCILE_POLL_INTERVAL` for cadence.
|
||||
- The current scheduler model is still a simple polling runner rather than a full generic jobs platform.
|
||||
- Manual `POST /api/providers/{providerID}/reconcile` and CLI reconcile remain supported.
|
||||
|
||||
### 2. Real sub2api Compatibility Is Verified on a Fresh Host, but Requires Explicit Operator Preparation
|
||||
- Real-host validation has now been executed against a fresh redeployed sub2api host.
|
||||
@@ -16,28 +17,19 @@ This document covers known limitations that operators should be aware of before
|
||||
- However, host initialization alone is not enough: operators must explicitly create ordinary users, keep reusable credentials, bind keys to the correct group, and satisfy the billing/subscription prerequisites documented in `docs/REAL_HOST_ACCEPTANCE_RUNBOOK.md`.
|
||||
- This is therefore no longer a code-compatibility blocker; it is an explicit operational prerequisite.
|
||||
|
||||
### 3. Access Module Not Fully Structured per Implementation Plan
|
||||
- The `access` package contains only `closure.go` (the combined close/validate logic).
|
||||
- `planner.go`, `subscription_service.go`, `self_service_checker.go` are not separately extracted.
|
||||
- All access logic is functional in `closure.go` but not split per the planned directory structure.
|
||||
|
||||
### 4. Reconcile Logic Inline in Provision Package
|
||||
- Reconcile lives in `internal/provision/batch_detail_and_reconcile_service.go` rather than a separate `internal/reconcile/*` package.
|
||||
- Functionally complete but structural gap vs implementation plan.
|
||||
|
||||
### 5. Standard Multi-stage Docker Build Still Depends on Outbound Module Download
|
||||
### 3. Standard Multi-stage Docker Build Still Depends on Outbound Module Download
|
||||
- `Dockerfile.local` has been validated as the recommended proxy-safe build path.
|
||||
- `scripts/build_local_image.sh` now prebuilds the Linux binary on the host and produces `sub2api-cn-relay-manager:local` reliably in this environment.
|
||||
- The standard multi-stage `Dockerfile` still requires outbound Go module download from inside the container build context; in restricted networks, prefer the local-image path.
|
||||
|
||||
## Accepted Design Trade-offs
|
||||
|
||||
### 6. CLI Run Functions Not Unit-Tested
|
||||
### 4. CLI Run Functions Not Unit-Tested
|
||||
- `runInstallPack`, `runImportProvider`, `runPreviewProvider`, `runRollbackProvider`, `runReconcileProvider`, `findProvider` connect to real SQLite/sub2api — these are 0% covered in unit tests.
|
||||
- The `execute()` dispatch and all `parse*` functions are fully tested.
|
||||
- In an integration/E2E context these functions are exercised through the host stub.
|
||||
|
||||
### 7. No Web UI
|
||||
### 5. No Web UI
|
||||
- Administration is through CLI and HTTP API only.
|
||||
- Consistent with MVP scope defined in PRD.
|
||||
|
||||
@@ -45,7 +37,9 @@ This document covers known limitations that operators should be aware of before
|
||||
|
||||
### Token Security
|
||||
- `SUB2API_CRM_ADMIN_TOKEN` must be at least 20 characters, rotated outside source control.
|
||||
- API keys imported via `--access-api-key` are used for gateway probe calls — they are not stored in control-plane state (only key fingerprint/hash is stored).
|
||||
- To support continuous background reconcile, the latest access closure now persists probe metadata in control-plane state:
|
||||
`self_service` stores the probe API key, and `subscription` stores the subscription user selector metadata.
|
||||
- Operators should therefore treat the SQLite database as secret-bearing state and protect it accordingly.
|
||||
|
||||
### Database
|
||||
- SQLite is the only supported database backend for v0.1.
|
||||
|
||||
@@ -86,9 +86,7 @@
|
||||
- `subscription` 需要 subscription 类型 group + 普通用户订阅分配 + key/group 绑定
|
||||
|
||||
### P2 已接受技术债务
|
||||
- access 模块仍未按 implementation plan 拆到 `planner.go / subscription_service.go / self_service_checker.go`
|
||||
- reconcile 仍内联在 `internal/provision/`,未拆到 `internal/reconcile/*`
|
||||
- 无内置 scheduler/jobs;当前通过手动 reconcile + 外部 cron 补偿
|
||||
- `internal/worker` 已抽出通用 polling runner,当前 batch-import runtime 与后台 reconcile 都已接入;调度模型仍是固定间隔 polling,而不是完整 jobs/reconcile 平台
|
||||
- CLI `run*` 真实链路函数未做系统性 mock 单测
|
||||
- 标准多阶段 `Dockerfile` 在受限网络下仍依赖容器内联网拉取 Go modules;本地部署默认走 `scripts/build_local_image.sh`
|
||||
- `subscription` 这条 provider matrix 已通过;剩余待补的是 latest-head `self_service` fresh-host 复验,而不是继续替换 provider key
|
||||
|
||||
@@ -173,6 +173,12 @@ paths:
|
||||
get:
|
||||
security:
|
||||
- bearerAuth: []
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/BatchImportRunStateQuery'
|
||||
- $ref: '#/components/parameters/BatchImportAccessModeQuery'
|
||||
- $ref: '#/components/parameters/BatchImportQuery'
|
||||
- $ref: '#/components/parameters/CursorQuery'
|
||||
- $ref: '#/components/parameters/LimitQuery'
|
||||
responses:
|
||||
'200':
|
||||
description: list batch import runs
|
||||
@@ -205,6 +211,16 @@ paths:
|
||||
- bearerAuth: []
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/RunID'
|
||||
- $ref: '#/components/parameters/BatchImportCurrentStageQuery'
|
||||
- $ref: '#/components/parameters/BatchImportConfirmationStatusQuery'
|
||||
- $ref: '#/components/parameters/BatchImportAccessStatusQuery'
|
||||
- $ref: '#/components/parameters/BatchImportHasWarningQuery'
|
||||
- $ref: '#/components/parameters/BatchImportProviderIDQuery'
|
||||
- $ref: '#/components/parameters/BatchImportMatchedAccountStateQuery'
|
||||
- $ref: '#/components/parameters/BatchImportAccountResolutionQuery'
|
||||
- $ref: '#/components/parameters/BatchImportQuery'
|
||||
- $ref: '#/components/parameters/CursorQuery'
|
||||
- $ref: '#/components/parameters/LimitQuery'
|
||||
responses:
|
||||
'200':
|
||||
description: batch import run items
|
||||
@@ -471,6 +487,86 @@ components:
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
BatchImportRunStateQuery:
|
||||
name: state
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
enum: [running, completed, completed_with_warnings, failed, cancelled]
|
||||
BatchImportAccessModeQuery:
|
||||
name: access_mode
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
enum: [subscription, self_service]
|
||||
BatchImportQuery:
|
||||
name: q
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
CursorQuery:
|
||||
name: cursor
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
LimitQuery:
|
||||
name: limit
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: integer
|
||||
minimum: 1
|
||||
BatchImportCurrentStageQuery:
|
||||
name: current_stage
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
enum: [probe, provision, confirm, validate, done]
|
||||
BatchImportConfirmationStatusQuery:
|
||||
name: confirmation_status
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
enum: [pending, confirmed, advisory, failed]
|
||||
BatchImportAccessStatusQuery:
|
||||
name: access_status
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
enum: [unknown, active, degraded, broken]
|
||||
BatchImportHasWarningQuery:
|
||||
name: has_warning
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: boolean
|
||||
BatchImportProviderIDQuery:
|
||||
name: provider_id
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
BatchImportMatchedAccountStateQuery:
|
||||
name: matched_account_state
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
enum: [none, active, disabled, deprecated, broken]
|
||||
BatchImportAccountResolutionQuery:
|
||||
name: account_resolution
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
enum: [created, reused, reactivated, replaced]
|
||||
responses:
|
||||
Unauthorized:
|
||||
description: missing or invalid admin token
|
||||
@@ -710,6 +806,9 @@ components:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/BatchImportRunSummary'
|
||||
next_cursor:
|
||||
type: string
|
||||
nullable: true
|
||||
BatchImportCapabilityTransportProfile:
|
||||
type: object
|
||||
properties:
|
||||
@@ -886,6 +985,9 @@ components:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/BatchImportRunItemSummary'
|
||||
next_cursor:
|
||||
type: string
|
||||
nullable: true
|
||||
ImportBatchInfo:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
@@ -0,0 +1,784 @@
|
||||
# Batch Auto-Import V2 Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** 实现 V2 的 URL + key 批量导入能力,覆盖模型发现、同模型别名归并、重复导入复用、异步确认、最终 gateway 验证、结果 API 与结果页所需状态投影。
|
||||
|
||||
**Architecture:** 采用 `BatchImportService + ConfirmationWorker + ValidationService + RunStateStore + ResultProjection` 分层架构。V2 只以 `import_runs / import_run_items / import_run_item_events` 作为运行态真相,旧 `import_batches/*` 仅保留 legacy linkage。重复导入决策基于 `provider_id + api_key_fingerprint + canonical_model_family`,最终可用性只认宿主真实 `/v1/chat/completions`。
|
||||
|
||||
**Tech Stack:** Go 1.22.2、`database/sql` + SQLite、Chi、OpenAPI 3.1、Go `testing`、`httptest`、现有 `internal/host/sub2api` 适配层与 `tests/integration` 集成测试套件。
|
||||
|
||||
---
|
||||
|
||||
## 0. 实施约束
|
||||
|
||||
- 只通过宿主 HTTP API 工作,不直写宿主数据库。
|
||||
- 所有状态枚举、字段名、API 路由必须遵循当前 canonical contract。
|
||||
- 每个任务都先写失败测试,再做最小实现,再跑验证。
|
||||
- 每个任务独立提交,避免大而混杂的 commit。
|
||||
- 任何 UI/API 展示都只能读 V2 canonical state,不得回退到日志拼接。
|
||||
|
||||
## 1. 任务总览
|
||||
|
||||
```text
|
||||
T1 Canonical types and enums
|
||||
T2 Probe models + alias normalization + canonical family
|
||||
T3 Capability profile + smoke completion routing
|
||||
T4 Provider ID + reuse policy
|
||||
T5 Run/item/event state store repositories
|
||||
T6 BatchImportService: Stage 0~2
|
||||
T7 ConfirmationWorker + retry + lease
|
||||
T8 ValidationService + access status
|
||||
T9 ResultProjection
|
||||
T10 HTTP API: runs/items
|
||||
T11 CLI: batch-import
|
||||
T12 Integration + contract verification
|
||||
T13 Design restoration audit
|
||||
```
|
||||
|
||||
## 2. 设计还原验证矩阵
|
||||
|
||||
### 2.1 目标覆盖矩阵
|
||||
|
||||
| 设计目标 | 对应任务 | 验证方式 |
|
||||
|---|---|---|
|
||||
| URL + key 自动发现模型 | T2, T6, T12 | `/v1/models` 拉取、集成测试 |
|
||||
| 模型纠错与别名归一化 | T2, T4, T9, T12 | unit + item detail projection |
|
||||
| 同模型跨中转快速识别 | T2, T4, T12 | `canonical_model_family` 测试 |
|
||||
| 重复导入自动复用 | T4, T6, T9, T12 | reuse decision + projection |
|
||||
| 已启用重复账号直接复用 | T4, T6, T9, T12 | `matched_account_state=active` |
|
||||
| 已停用/已弃用账号快速启用 | T4, T6, T7, T9, T12 | `account_resolution=reactivated` |
|
||||
| transport + model capability profile | T3, T9, T10, T12 | profile persistence + API schema |
|
||||
| channel/account 演化 | T6, T12 | patch contract + host stub |
|
||||
| 异步确认与重试 | T7, T12 | lease/retry/event trail |
|
||||
| gateway completion 最终判定 | T8, T12 | `access_status` 唯一写入 |
|
||||
| 结果 API 与结果页数据源 | T5, T9, T10, T12 | run/item/event projection |
|
||||
| 单一状态源 | T5, T7, T8, T9 | 只读 `import_runs/*` |
|
||||
|
||||
### 2.2 契约覆盖矩阵
|
||||
|
||||
| 契约 | 对应任务 |
|
||||
|---|---|
|
||||
| `run_id / item_id / provider_id` | T1, T4, T5 |
|
||||
| `run.state` | T1, T5, T9 |
|
||||
| `current_stage / confirmation_status / access_status` | T1, T5, T7, T8 |
|
||||
| `matched_account_state / account_resolution` | T4, T5, T6, T9, T10 |
|
||||
| `api_key_fingerprint` | T4, T5, T6 |
|
||||
| `canonical_model_families` | T2, T4, T5, T9, T10 |
|
||||
| `provision_reused / reused_from_*` | T4, T5, T6, T9, T10 |
|
||||
| `/api/batch-import/runs*` | T10, T12 |
|
||||
|
||||
如果 T1~T12 全部完成并通过验证,T13 必须能证明上述矩阵全部为“已覆盖”,否则不得宣称 V2 可按设计实现。
|
||||
|
||||
## 3. 实施任务
|
||||
|
||||
### Task 1: Canonical Types And Enums
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/batch/types.go`
|
||||
- Test: `internal/batch/types_test.go`
|
||||
- Reference: `docs/2026-05-21-BATCH_AUTO_IMPORT_SPEC.md`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
为以下枚举写失败测试:
|
||||
- `RunState`
|
||||
- `ItemStage`
|
||||
- `ConfirmationStatus`
|
||||
- `AccessStatus`
|
||||
- `MatchedAccountState`
|
||||
- `AccountResolution`
|
||||
|
||||
至少覆盖:
|
||||
- 常量值是否与文档一致
|
||||
- 非法字符串是否会在后续解析层被拒绝
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestRunStateConstants|TestItemStateConstants' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL,提示类型或常量不存在。
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
在 `internal/batch/types.go` 中定义上述类型与常量,不提前引入不需要的 helper。
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestRunStateConstants|TestItemStateConstants' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/batch/types.go internal/batch/types_test.go
|
||||
git commit -m "feat(batch): add canonical v2 state enums"
|
||||
```
|
||||
|
||||
### Task 2: Probe Models, Alias Normalization, Canonical Family
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/probe/models.go`
|
||||
- Create: `internal/probe/aliases.go`
|
||||
- Test: `internal/probe/models_test.go`
|
||||
- Test: `internal/probe/aliases_test.go`
|
||||
- Reference: `docs/2026-05-21-BATCH_AUTO_IMPORT_TDD_PLAN.md`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
覆盖:
|
||||
- `/v1/models` OpenAI 格式解析
|
||||
- 空模型列表
|
||||
- 鉴权失败
|
||||
- `kimi 2.6 / kimi-2.6 / kimi-k2.6` 归并到同一 `canonical_model_family`
|
||||
- `deepseek-ai/DeepSeek-V4-Pro` vendor 前缀归一化
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/probe -run 'TestProviderModels|TestCanonicalModelFamily' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
实现:
|
||||
- `ProviderModels`
|
||||
- `NormalizeModelID`
|
||||
- `CanonicalModelID`
|
||||
- `CanonicalModelFamily`
|
||||
- `BuildAliasTable`
|
||||
- `ResolveRequestedModel`
|
||||
- `RecommendModels`
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/probe -run 'TestProviderModels|TestCanonicalModelFamily' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/probe/models.go internal/probe/aliases.go internal/probe/models_test.go internal/probe/aliases_test.go
|
||||
git commit -m "feat(probe): add model discovery and canonical family normalization"
|
||||
```
|
||||
|
||||
### Task 3: Capability Profile And Smoke Completion Routing
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/probe/capability.go`
|
||||
- Create: `internal/probe/completion.go`
|
||||
- Test: `internal/probe/capability_test.go`
|
||||
- Test: `internal/probe/completion_test.go`
|
||||
- Reference: `docs/2026-05-22-BATCH_AUTO_IMPORT_V2_ARCHITECTURE.md`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
覆盖:
|
||||
- `responses` 不支持但 `chat/completions` 可用
|
||||
- transport profile 的 advisory 记录
|
||||
- per-model profile 记录
|
||||
- `ResolveSmokeModel` 基于别名与能力选择 smoke model
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/probe -run 'TestProbeCapabilities|TestResolveSmokeModel|TestSmokeCompletion' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
实现:
|
||||
- `TransportProfile`
|
||||
- `ModelCapabilityProfile`
|
||||
- `CapabilityProfile`
|
||||
- `ProbeCapabilities`
|
||||
- `CompletionResult`
|
||||
- `ResolveSmokeModel`
|
||||
- `SmokeCompletion`
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/probe -run 'TestProbeCapabilities|TestResolveSmokeModel|TestSmokeCompletion' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/probe/capability.go internal/probe/completion.go internal/probe/capability_test.go internal/probe/completion_test.go
|
||||
git commit -m "feat(probe): add capability profile and smoke completion routing"
|
||||
```
|
||||
|
||||
### Task 4: Provider ID And Reuse Policy
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/batch/provider_id.go`
|
||||
- Create: `internal/batch/reuse_policy.go`
|
||||
- Test: `internal/batch/provider_id_test.go`
|
||||
- Test: `internal/batch/reuse_policy_test.go`
|
||||
- Reference: `docs/2026-05-21-BATCH_AUTO_IMPORT_SPEC.md:336`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
覆盖:
|
||||
- 同 host 不同 path 生成不同 `provider_id`
|
||||
- 已存在 active provider 且 family 已覆盖 -> `reused`
|
||||
- 已存在 active account -> `matched_account_state=active`, `account_resolution=reused`
|
||||
- `disabled/deprecated` 账号 -> `reactivated`
|
||||
- `broken` provider/account -> `replace`
|
||||
- 同 family 不同 alias -> 视为已覆盖
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestNormalizeProviderID|TestDecideReuse' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
实现:
|
||||
- `NormalizeProviderID`
|
||||
- `ReuseDecision`
|
||||
- `DecideReuse`
|
||||
|
||||
不要在这一步直接改 service。
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestNormalizeProviderID|TestDecideReuse' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/batch/provider_id.go internal/batch/reuse_policy.go internal/batch/provider_id_test.go internal/batch/reuse_policy_test.go
|
||||
git commit -m "feat(batch): add provider id and reuse policy"
|
||||
```
|
||||
|
||||
### Task 5: Run/Item/Event State Store Repositories
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/store/migrations/0007_batch_import_runs.sql`
|
||||
- Modify: `internal/store/migrations/0008_batch_import_run_events.sql`
|
||||
- Modify: `internal/store/sqlite/import_runs_repo.go`
|
||||
- Create: `internal/store/sqlite/import_run_items_repo.go`
|
||||
- Create: `internal/store/sqlite/import_run_item_events_repo.go`
|
||||
- Modify: `internal/store/sqlite/db.go`
|
||||
- Test: `internal/store/sqlite/import_runs_repo_test.go`
|
||||
- Test: `tests/integration/store_init_test.go`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
覆盖:
|
||||
- run 创建/更新
|
||||
- item upsert 持久化 `api_key_fingerprint / canonical_model_families / matched_account_state / account_resolution / provision_reused`
|
||||
- event append/list
|
||||
- lease 字段持久化
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/store/sqlite/... ./tests/integration/... -run 'TestRunStateStore|TestStoreAppliesLatestMigration' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
补足 repo 与 migration,确保 schema 与文档完全一致。
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/store/sqlite/... ./tests/integration/... -run 'TestRunStateStore|TestStoreAppliesLatestMigration' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/store/migrations/0007_batch_import_runs.sql internal/store/migrations/0008_batch_import_run_events.sql internal/store/sqlite/import_runs_repo.go internal/store/sqlite/import_run_items_repo.go internal/store/sqlite/import_run_item_events_repo.go internal/store/sqlite/db.go internal/store/sqlite/import_runs_repo_test.go tests/integration/store_init_test.go
|
||||
git commit -m "feat(store): complete v2 runtime state repositories"
|
||||
```
|
||||
|
||||
### Task 6: BatchImportService Stage 0~2
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/batch/service.go`
|
||||
- Create: `internal/batch/capability_profile.go`
|
||||
- Create: `internal/batch/channel_evolution.go`
|
||||
- Test: `internal/batch/service_test.go`
|
||||
- Test: `internal/batch/channel_evolution_test.go`
|
||||
- Reference: `internal/provision/import_service.go`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
覆盖:
|
||||
- 创建 run + items
|
||||
- reuse preflight 跳过重复 provision
|
||||
- active 账号重复导入 -> reused
|
||||
- deprecated 账号重复导入 -> reactivated
|
||||
- patch-only 新 alias
|
||||
- legacy batch/provider link 回写
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestBatchImport_StartRun|TestModelMappingDelta' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
实现:
|
||||
- `BatchImportService.StartRun`
|
||||
- `ImportRoutingStrategy`
|
||||
- `BuildImportRoutingStrategy`
|
||||
- `ChannelPatchContract`
|
||||
- `ModelMappingDelta`
|
||||
|
||||
先接现有 `provision.ImportService`,不要提前扩展 UI/API。
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestBatchImport_StartRun|TestModelMappingDelta' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/batch/service.go internal/batch/capability_profile.go internal/batch/channel_evolution.go internal/batch/service_test.go internal/batch/channel_evolution_test.go
|
||||
git commit -m "feat(batch): implement v2 run setup and provision stages"
|
||||
```
|
||||
|
||||
### Task 7: ConfirmationWorker, Lease And Retry
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/batch/confirmation.go`
|
||||
- Test: `internal/batch/confirmation_test.go`
|
||||
- Reference: `docs/2026-05-22-BATCH_AUTO_IMPORT_V2_ARCHITECTURE.md:398`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
覆盖:
|
||||
- 只捞 `confirm + pending + retry_due + lease_expired`
|
||||
- `403` probe race -> advisory
|
||||
- 初次 `503 no available accounts` -> retry -> success
|
||||
- 多 worker lease 互斥
|
||||
- `disabled/deprecated` 命中后 reactivated 投影正确
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestConfirmationWorker' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
实现:
|
||||
- `ConfirmationWorker.Tick`
|
||||
- `ConfirmationWorker.ConfirmItem`
|
||||
- retry 计划
|
||||
- lease 生命周期
|
||||
- advisory event 写入
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestConfirmationWorker' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/batch/confirmation.go internal/batch/confirmation_test.go
|
||||
git commit -m "feat(batch): add confirmation worker and retry handling"
|
||||
```
|
||||
|
||||
### Task 8: ValidationService And Final Access Status
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/batch/validation.go`
|
||||
- Test: `internal/batch/validation_test.go`
|
||||
- Reference: `internal/access/closure.go`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
覆盖:
|
||||
- `confirmed/advisory + chat 200 -> active`
|
||||
- exhausted transient -> `degraded`
|
||||
- definitive invalid path -> `broken`
|
||||
- 只有 ValidationService 可以写 `access_status`
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestValidationService' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
实现:
|
||||
- `ValidationService.ValidateItem`
|
||||
- `access_status` 映射
|
||||
- 对 run summary 的最小更新
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestValidationService' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/batch/validation.go internal/batch/validation_test.go
|
||||
git commit -m "feat(batch): add validation service for final access status"
|
||||
```
|
||||
|
||||
### Task 9: ResultProjection
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/batch/status_projection.go`
|
||||
- Test: `internal/batch/status_projection_test.go`
|
||||
- Reference: `docs/2026-05-22-BATCH_AUTO_IMPORT_V2_API_SCHEMAS.md`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
覆盖:
|
||||
- run summary 聚合
|
||||
- item summary/detail projection
|
||||
- warning 文案模板
|
||||
- `provision_reused` badge
|
||||
- `matched_account_state / account_resolution` 文案与 badge
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestStatusProjection' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
实现:
|
||||
- run list projection
|
||||
- item list projection
|
||||
- item detail projection
|
||||
- warning/badge mapping
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/batch -run 'TestStatusProjection' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/batch/status_projection.go internal/batch/status_projection_test.go
|
||||
git commit -m "feat(batch): add result projection for v2 runs and items"
|
||||
```
|
||||
|
||||
### Task 10: HTTP API For Runs And Items
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/app/http_batch_import.go`
|
||||
- Create: `internal/app/http_batch_runs.go`
|
||||
- Modify: `internal/app/http_api.go`
|
||||
- Test: `internal/app/http_batch_import_test.go`
|
||||
- Test: `internal/app/http_batch_runs_test.go`
|
||||
- Reference: `docs/openapi.yaml`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
覆盖:
|
||||
- `POST /api/batch-import/runs`
|
||||
- `GET /api/batch-import/runs`
|
||||
- `GET /api/batch-import/runs/{run_id}`
|
||||
- `GET /api/batch-import/runs/{run_id}/items`
|
||||
- `GET /api/batch-import/runs/{run_id}/items/{item_id}`
|
||||
- `subscription/self_service` 条件必填
|
||||
- 列表过滤 `matched_account_state / account_resolution`
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/app -run 'TestBatchImportHTTP|TestBatchRunsHTTP' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
按 OpenAPI 只输出 projection,不泄漏 legacy 表结构。
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./internal/app -run 'TestBatchImportHTTP|TestBatchRunsHTTP' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/app/http_batch_import.go internal/app/http_batch_runs.go internal/app/http_api.go internal/app/http_batch_import_test.go internal/app/http_batch_runs_test.go
|
||||
git commit -m "feat(api): add batch import v2 endpoints"
|
||||
```
|
||||
|
||||
### Task 11: CLI Entry For Batch Import
|
||||
|
||||
**Files:**
|
||||
- Modify: `cmd/cli/main.go`
|
||||
- Create: `cmd/cli/batch_import.go`
|
||||
- Test: `cmd/cli/batch_import_test.go`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
覆盖:
|
||||
- 参数解析
|
||||
- `subscription` 必填订阅参数
|
||||
- `self_service` 必填 `probe_api_key`
|
||||
- `--confirm-timeout`
|
||||
- 结果输出 `run_id/result_page`
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./cmd/cli -run 'TestBatchImportCLI' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
实现 CLI 到 V2 API/service 的入口,不在 CLI 层重复业务逻辑。
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./cmd/cli -run 'TestBatchImportCLI' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add cmd/cli/main.go cmd/cli/batch_import.go cmd/cli/batch_import_test.go
|
||||
git commit -m "feat(cli): add v2 batch import command"
|
||||
```
|
||||
|
||||
### Task 12: Integration And End-To-End Verification
|
||||
|
||||
**Files:**
|
||||
- Create: `tests/integration/batch_import_v2_test.go`
|
||||
- Modify: `tests/integration/host_stub_test.go`(如需 stub 扩展)
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
至少覆盖 6 条真实业务链:
|
||||
- 发现模型并归一化
|
||||
- 重复导入 active 账号 -> reused
|
||||
- deprecated 账号 -> reactivated
|
||||
- 同 family 不同 alias -> patch_only
|
||||
- probe race + warmup retry -> advisory + active
|
||||
- run/item/event 详情可从 V2 新表完全读出
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./tests/integration/... -run 'TestBatchImportV2' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
补齐 host stub、fake adapter、seed data,确保每条链路都可复现。
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./tests/integration/... -run 'TestBatchImportV2' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add tests/integration/batch_import_v2_test.go tests/integration/host_stub_test.go
|
||||
git commit -m "test(integration): cover batch import v2 flows"
|
||||
```
|
||||
|
||||
### Task 13: Design Restoration Audit
|
||||
|
||||
**Files:**
|
||||
- Create: `docs/2026-05-22-BATCH_AUTO_IMPORT_V2_RESTORATION_CHECKLIST.md`
|
||||
- Modify: `docs/EXECUTION_BOARD.md`
|
||||
|
||||
**Step 1: Write the failing audit checklist**
|
||||
|
||||
列出必须逐项勾选的设计恢复项:
|
||||
- 8 项 Objective
|
||||
- canonical contract
|
||||
- 结果 API
|
||||
- migration
|
||||
- worker/retry/lease
|
||||
- reuse/reactivation
|
||||
|
||||
**Step 2: Run verification to identify gaps**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
go test ./... -count=1
|
||||
go test ./tests/integration/... -count=1
|
||||
go test -cover ./internal/... -count=1
|
||||
go vet ./...
|
||||
gofmt -l .
|
||||
```
|
||||
|
||||
Expected: 在实现完成前,这一步用来发现剩余设计缺口;在最终完成时必须全绿。
|
||||
|
||||
**Step 3: Write the audit artifact**
|
||||
|
||||
将每一项设计要求映射到:
|
||||
- 代码文件
|
||||
- 测试文件
|
||||
- API 路由
|
||||
- 状态字段
|
||||
|
||||
**Step 4: Update board with true gate**
|
||||
|
||||
在执行板中明确:
|
||||
- 哪些任务完成
|
||||
- 哪些设计要求已还原
|
||||
- 是否可宣称“V2 设计已被完整实现”
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add docs/2026-05-22-BATCH_AUTO_IMPORT_V2_RESTORATION_CHECKLIST.md docs/EXECUTION_BOARD.md
|
||||
git commit -m "docs(v2): add restoration checklist and completion gate"
|
||||
```
|
||||
|
||||
## 4. 全局验证门禁
|
||||
|
||||
完成 T1~T13 后,必须一次性通过:
|
||||
|
||||
```bash
|
||||
gofmt -l .
|
||||
go vet ./...
|
||||
go test ./... -count=1
|
||||
go test ./tests/integration/... -count=1
|
||||
go test -cover ./internal/... -count=1
|
||||
```
|
||||
|
||||
额外检查:
|
||||
|
||||
- `docs/openapi.yaml` 与 handler 响应字段一致
|
||||
- `import_runs/*` 足以支撑结果页,不依赖 legacy 表拼接
|
||||
- `matched_account_state / account_resolution / provision_reused` 能在 item detail 里直接读到
|
||||
- `canonical_model_family` 能把同模型别名判定为同一族
|
||||
|
||||
## 5. 计划完整性结论
|
||||
|
||||
这份计划只有在满足以下条件时,才算“任务可以完全还原规划设计”:
|
||||
|
||||
1. T1~T12 实现完成并全部通过验证
|
||||
2. T13 的还原清单中不存在未映射设计项
|
||||
3. 任一 Objective 都能指向至少一条:
|
||||
- 实现任务
|
||||
- 自动化测试
|
||||
- API 或状态字段证据
|
||||
4. 结果页/API 不需要额外新增未规划字段才能解释最终状态
|
||||
|
||||
如果 T13 审核时发现任何一项设计要求无法映射到任务或测试,这份计划必须回退修改,不能直接进入实现。
|
||||
|
||||
## 6. 推荐提交顺序
|
||||
|
||||
建议按以下小步提交:
|
||||
|
||||
1. `feat(batch): add canonical v2 state enums`
|
||||
2. `feat(probe): add model discovery and canonical family normalization`
|
||||
3. `feat(probe): add capability profile and smoke completion routing`
|
||||
4. `feat(batch): add provider id and reuse policy`
|
||||
5. `feat(store): complete v2 runtime state repositories`
|
||||
6. `feat(batch): implement v2 run setup and provision stages`
|
||||
7. `feat(batch): add confirmation worker and retry handling`
|
||||
8. `feat(batch): add validation service for final access status`
|
||||
9. `feat(batch): add result projection for v2 runs and items`
|
||||
10. `feat(api): add batch import v2 endpoints`
|
||||
11. `feat(cli): add v2 batch import command`
|
||||
12. `test(integration): cover batch import v2 flows`
|
||||
13. `docs(v2): add restoration checklist and completion gate`
|
||||
|
||||
Plan complete and saved to `docs/plans/2026-05-22-batch-auto-import-v2-implementation-plan.md`. Two execution options:
|
||||
|
||||
**1. Subagent-Driven (this session)** - I dispatch fresh subagent per task, review between tasks, fast iteration
|
||||
|
||||
**2. Parallel Session (separate)** - Open new session with executing-plans, batch execution with checkpoints
|
||||
|
||||
Which approach?
|
||||
@@ -1,154 +0,0 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
)
|
||||
|
||||
const (
|
||||
ModeSubscription = "subscription"
|
||||
ModeSelfService = "self_service"
|
||||
|
||||
gatewayCompletionRetryAttempts = 3
|
||||
gatewayCompletionRetryDelay = 300 * time.Millisecond
|
||||
)
|
||||
|
||||
type SubscriptionTarget struct {
|
||||
UserID string
|
||||
DurationDays int
|
||||
}
|
||||
|
||||
type ClosureRequest struct {
|
||||
Mode string
|
||||
ProbeAPIKey string
|
||||
Subscriptions []SubscriptionTarget
|
||||
GroupID string
|
||||
ExpectedModel string
|
||||
Prompt string
|
||||
MaxTokens int
|
||||
}
|
||||
|
||||
type Host interface {
|
||||
EnsureSubscriptionAccess(ctx context.Context, req sub2api.EnsureSubscriptionAccessRequest) (sub2api.SubscriptionAccessRef, error)
|
||||
AssignSubscription(ctx context.Context, req sub2api.AssignSubscriptionRequest) (sub2api.SubscriptionRef, error)
|
||||
CheckGatewayAccess(ctx context.Context, req sub2api.GatewayAccessCheckRequest) (sub2api.GatewayAccessResult, error)
|
||||
CheckGatewayCompletion(ctx context.Context, req sub2api.GatewayCompletionCheckRequest) (sub2api.GatewayCompletionResult, error)
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
host Host
|
||||
}
|
||||
|
||||
func NewService(host Host) *Service {
|
||||
return &Service{host: host}
|
||||
}
|
||||
|
||||
func Validate(req ClosureRequest) error {
|
||||
switch strings.TrimSpace(req.Mode) {
|
||||
case ModeSubscription:
|
||||
if len(req.Subscriptions) == 0 {
|
||||
return fmt.Errorf("subscription access requires at least one subscription target")
|
||||
}
|
||||
case ModeSelfService:
|
||||
if strings.TrimSpace(req.ProbeAPIKey) == "" {
|
||||
return fmt.Errorf("self_service access requires probe api key")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported access mode %q", req.Mode)
|
||||
}
|
||||
if strings.TrimSpace(req.Mode) != ModeSubscription && strings.TrimSpace(req.ProbeAPIKey) == "" {
|
||||
return fmt.Errorf("access probe api key is required to verify gateway closure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) Close(ctx context.Context, req ClosureRequest) (sub2api.GatewayAccessResult, error) {
|
||||
if s == nil || s.host == nil {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("access host is required")
|
||||
}
|
||||
if err := Validate(req); err != nil {
|
||||
return sub2api.GatewayAccessResult{}, err
|
||||
}
|
||||
probeAPIKey := strings.TrimSpace(req.ProbeAPIKey)
|
||||
if strings.TrimSpace(req.Mode) == ModeSubscription {
|
||||
for _, target := range req.Subscriptions {
|
||||
resolvedTarget := target.UserID
|
||||
accessRef, err := s.host.EnsureSubscriptionAccess(ctx, sub2api.EnsureSubscriptionAccessRequest{UserSelector: target.UserID, GroupID: req.GroupID})
|
||||
if err != nil {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("ensure subscription access for %s: %w", target.UserID, err)
|
||||
}
|
||||
if strings.TrimSpace(accessRef.UserID) != "" {
|
||||
resolvedTarget = accessRef.UserID
|
||||
}
|
||||
if strings.TrimSpace(accessRef.APIKey) != "" {
|
||||
probeAPIKey = strings.TrimSpace(accessRef.APIKey)
|
||||
}
|
||||
if _, err := s.host.AssignSubscription(ctx, sub2api.AssignSubscriptionRequest{UserID: resolvedTarget, GroupID: req.GroupID, DurationDays: target.DurationDays}); err != nil {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("assign subscription for %s: %w", target.UserID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if probeAPIKey == "" {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("access probe api key is required to verify gateway closure")
|
||||
}
|
||||
result, err := s.host.CheckGatewayAccess(ctx, sub2api.GatewayAccessCheckRequest{APIKey: probeAPIKey, ExpectedModel: req.ExpectedModel})
|
||||
if err != nil {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("check gateway access: %w", err)
|
||||
}
|
||||
if result.OK && result.HasExpectedModel && strings.TrimSpace(req.ExpectedModel) != "" {
|
||||
completion, err := s.checkGatewayCompletionWithRetry(ctx, sub2api.GatewayCompletionCheckRequest{
|
||||
APIKey: probeAPIKey,
|
||||
Model: req.ExpectedModel,
|
||||
Prompt: req.Prompt,
|
||||
MaxTokens: req.MaxTokens,
|
||||
})
|
||||
if err != nil {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("check gateway completion: %w", err)
|
||||
}
|
||||
result.CompletionOK = completion.OK
|
||||
result.CompletionStatus = completion.StatusCode
|
||||
result.CompletionType = completion.ContentType
|
||||
result.CompletionBody = completion.BodyPreview
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) checkGatewayCompletionWithRetry(ctx context.Context, req sub2api.GatewayCompletionCheckRequest) (sub2api.GatewayCompletionResult, error) {
|
||||
var last sub2api.GatewayCompletionResult
|
||||
for attempt := 1; attempt <= gatewayCompletionRetryAttempts; attempt++ {
|
||||
completion, err := s.host.CheckGatewayCompletion(ctx, req)
|
||||
if err != nil {
|
||||
return sub2api.GatewayCompletionResult{}, err
|
||||
}
|
||||
last = completion
|
||||
if completion.OK || !isTransientGatewayCompletionFailure(completion) || attempt == gatewayCompletionRetryAttempts {
|
||||
return completion, nil
|
||||
}
|
||||
timer := time.NewTimer(gatewayCompletionRetryDelay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return last, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return last, nil
|
||||
}
|
||||
|
||||
func isTransientGatewayCompletionFailure(result sub2api.GatewayCompletionResult) bool {
|
||||
if result.OK {
|
||||
return false
|
||||
}
|
||||
if result.StatusCode != 0 && result.StatusCode != 429 && result.StatusCode != 502 && result.StatusCode != 503 && result.StatusCode != 504 {
|
||||
return false
|
||||
}
|
||||
body := strings.ToLower(strings.TrimSpace(result.BodyPreview))
|
||||
return strings.Contains(body, "service temporarily unavailable") ||
|
||||
strings.Contains(body, "no available accounts") ||
|
||||
strings.Contains(body, "temporar") ||
|
||||
strings.Contains(body, "try again")
|
||||
}
|
||||
@@ -143,18 +143,55 @@ func TestServiceCloseRetriesTransientGatewayCompletionFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceCloseRepairsOpenAIResponsesCapabilityMismatch(t *testing.T) {
|
||||
host := &fakeClosureHost{
|
||||
gatewayResult: sub2api.GatewayAccessResult{OK: true, StatusCode: 200, HasExpectedModel: true, Models: []string{"kimi-k2.6"}},
|
||||
completionResults: []sub2api.GatewayCompletionResult{
|
||||
{OK: false, StatusCode: 502, ContentType: "application/json", BodyPreview: `{"error":{"message":"Upstream service temporarily unavailable","type":"upstream_error"}}`},
|
||||
},
|
||||
completionAfterRepair: &sub2api.GatewayCompletionResult{OK: true, StatusCode: 200, ContentType: "application/json"},
|
||||
managedAccess: map[string]sub2api.SubscriptionAccessRef{
|
||||
"user-1": {UserID: "host-user-1", APIKey: "managed-user-key"},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := NewService(host).Close(context.Background(), ClosureRequest{
|
||||
Mode: "subscription",
|
||||
GroupID: "group-1",
|
||||
AccountIDs: []string{"account-1", "account-1"},
|
||||
ExpectedModel: "kimi-k2.6",
|
||||
ResponsesCapabilitySuspect: true,
|
||||
Subscriptions: []SubscriptionTarget{{UserID: "user-1", DurationDays: 30}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
if !result.CompletionOK || result.CompletionStatus != 200 {
|
||||
t.Fatalf("completion result = %+v, want repaired success", result)
|
||||
}
|
||||
if host.disableResponsesCalls != 1 {
|
||||
t.Fatalf("disable responses calls = %d, want 1", host.disableResponsesCalls)
|
||||
}
|
||||
if len(host.disabledResponsesAccountIDs) != 1 || host.disabledResponsesAccountIDs[0] != "account-1" {
|
||||
t.Fatalf("disabled responses account ids = %v, want [account-1]", host.disabledResponsesAccountIDs)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeClosureHost struct {
|
||||
assigned []sub2api.AssignSubscriptionRequest
|
||||
managedAccess map[string]sub2api.SubscriptionAccessRef
|
||||
assignErr error
|
||||
gatewayProbe sub2api.GatewayAccessCheckRequest
|
||||
gatewayResult sub2api.GatewayAccessResult
|
||||
gatewayErr error
|
||||
completionProbe sub2api.GatewayCompletionCheckRequest
|
||||
completionCalls int
|
||||
completionResults []sub2api.GatewayCompletionResult
|
||||
completionResult sub2api.GatewayCompletionResult
|
||||
completionErr error
|
||||
assigned []sub2api.AssignSubscriptionRequest
|
||||
managedAccess map[string]sub2api.SubscriptionAccessRef
|
||||
assignErr error
|
||||
gatewayProbe sub2api.GatewayAccessCheckRequest
|
||||
gatewayResult sub2api.GatewayAccessResult
|
||||
gatewayErr error
|
||||
completionProbe sub2api.GatewayCompletionCheckRequest
|
||||
completionCalls int
|
||||
completionResults []sub2api.GatewayCompletionResult
|
||||
completionResult sub2api.GatewayCompletionResult
|
||||
completionAfterRepair *sub2api.GatewayCompletionResult
|
||||
completionErr error
|
||||
disableResponsesCalls int
|
||||
disabledResponsesAccountIDs []string
|
||||
}
|
||||
|
||||
func (f *fakeClosureHost) EnsureSubscriptionAccess(_ context.Context, req sub2api.EnsureSubscriptionAccessRequest) (sub2api.SubscriptionAccessRef, error) {
|
||||
@@ -186,6 +223,9 @@ func (f *fakeClosureHost) CheckGatewayCompletion(_ context.Context, req sub2api.
|
||||
if f.completionErr != nil {
|
||||
return sub2api.GatewayCompletionResult{}, f.completionErr
|
||||
}
|
||||
if f.disableResponsesCalls > 0 && f.completionAfterRepair != nil {
|
||||
return *f.completionAfterRepair, nil
|
||||
}
|
||||
if len(f.completionResults) > 0 {
|
||||
idx := f.completionCalls - 1
|
||||
if idx >= len(f.completionResults) {
|
||||
@@ -195,3 +235,9 @@ func (f *fakeClosureHost) CheckGatewayCompletion(_ context.Context, req sub2api.
|
||||
}
|
||||
return f.completionResult, nil
|
||||
}
|
||||
|
||||
func (f *fakeClosureHost) DisableOpenAIResponsesAPI(_ context.Context, accountIDs []string) error {
|
||||
f.disableResponsesCalls++
|
||||
f.disabledResponsesAccountIDs = append([]string(nil), accountIDs...)
|
||||
return nil
|
||||
}
|
||||
|
||||
80
internal/access/gateway_validation.go
Normal file
80
internal/access/gateway_validation.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
)
|
||||
|
||||
func (s *Service) verifyGatewayClosure(ctx context.Context, req ClosureRequest, plan closurePlan) (sub2api.GatewayAccessResult, error) {
|
||||
if plan.probeAPIKey == "" {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("access probe api key is required to verify gateway closure")
|
||||
}
|
||||
result, err := s.host.CheckGatewayAccess(ctx, sub2api.GatewayAccessCheckRequest{
|
||||
APIKey: plan.probeAPIKey,
|
||||
ExpectedModel: req.ExpectedModel,
|
||||
})
|
||||
if err != nil {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("check gateway access: %w", err)
|
||||
}
|
||||
if result.OK && result.HasExpectedModel && strings.TrimSpace(req.ExpectedModel) != "" {
|
||||
completionReq := sub2api.GatewayCompletionCheckRequest{
|
||||
APIKey: plan.probeAPIKey,
|
||||
Model: req.ExpectedModel,
|
||||
Prompt: req.Prompt,
|
||||
MaxTokens: req.MaxTokens,
|
||||
}
|
||||
completion, err := s.checkGatewayCompletionWithRetry(ctx, completionReq)
|
||||
if err != nil {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("check gateway completion: %w", err)
|
||||
}
|
||||
completion, err = s.maybeRepairOpenAIResponsesCapability(ctx, req, completionReq, completion)
|
||||
if err != nil {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("re-check gateway completion after capability repair: %w", err)
|
||||
}
|
||||
result.CompletionOK = completion.OK
|
||||
result.CompletionStatus = completion.StatusCode
|
||||
result.CompletionType = completion.ContentType
|
||||
result.CompletionBody = completion.BodyPreview
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) checkGatewayCompletionWithRetry(ctx context.Context, req sub2api.GatewayCompletionCheckRequest) (sub2api.GatewayCompletionResult, error) {
|
||||
var last sub2api.GatewayCompletionResult
|
||||
for attempt := 1; attempt <= gatewayCompletionRetryAttempts; attempt++ {
|
||||
completion, err := s.host.CheckGatewayCompletion(ctx, req)
|
||||
if err != nil {
|
||||
return sub2api.GatewayCompletionResult{}, err
|
||||
}
|
||||
last = completion
|
||||
if completion.OK || !isTransientGatewayCompletionFailure(completion) || attempt == gatewayCompletionRetryAttempts {
|
||||
return completion, nil
|
||||
}
|
||||
timer := time.NewTimer(gatewayCompletionRetryDelay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return last, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return last, nil
|
||||
}
|
||||
|
||||
func isTransientGatewayCompletionFailure(result sub2api.GatewayCompletionResult) bool {
|
||||
if result.OK {
|
||||
return false
|
||||
}
|
||||
if result.StatusCode != 0 && result.StatusCode != 429 && result.StatusCode != 502 && result.StatusCode != 503 && result.StatusCode != 504 {
|
||||
return false
|
||||
}
|
||||
body := strings.ToLower(strings.TrimSpace(result.BodyPreview))
|
||||
return strings.Contains(body, "service temporarily unavailable") ||
|
||||
strings.Contains(body, "no available accounts") ||
|
||||
strings.Contains(body, "temporar") ||
|
||||
strings.Contains(body, "try again")
|
||||
}
|
||||
69
internal/access/openai_responses_repair.go
Normal file
69
internal/access/openai_responses_repair.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
)
|
||||
|
||||
func SuspectsOpenAIResponsesCapabilityMismatch(probe sub2api.ProbeResult) bool {
|
||||
if probe.OK {
|
||||
return false
|
||||
}
|
||||
message := strings.ToLower(strings.TrimSpace(probe.Message))
|
||||
if message == "" {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(message, "api returned 403: forbidden") {
|
||||
return true
|
||||
}
|
||||
return strings.Contains(message, "responses api") &&
|
||||
(strings.Contains(message, "当前测试接口仅支持") ||
|
||||
strings.Contains(message, "账号本身可正常使用") ||
|
||||
strings.Contains(message, "please directly") ||
|
||||
strings.Contains(message, "actual api"))
|
||||
}
|
||||
|
||||
func ShouldAttemptOpenAIResponsesCapabilityRepair(suspect bool, completion sub2api.GatewayCompletionResult) bool {
|
||||
if !suspect || completion.OK {
|
||||
return false
|
||||
}
|
||||
if completion.StatusCode != 502 && completion.StatusCode != 503 {
|
||||
return false
|
||||
}
|
||||
body := strings.ToLower(strings.TrimSpace(completion.BodyPreview))
|
||||
return strings.Contains(body, "service temporarily unavailable") ||
|
||||
strings.Contains(body, "no available accounts")
|
||||
}
|
||||
|
||||
func (s *Service) maybeRepairOpenAIResponsesCapability(ctx context.Context, req ClosureRequest, completionReq sub2api.GatewayCompletionCheckRequest, completion sub2api.GatewayCompletionResult) (sub2api.GatewayCompletionResult, error) {
|
||||
if !ShouldAttemptOpenAIResponsesCapabilityRepair(req.ResponsesCapabilitySuspect, completion) {
|
||||
return completion, nil
|
||||
}
|
||||
accountIDs := normalizedAccountIDs(req.AccountIDs)
|
||||
if len(accountIDs) == 0 {
|
||||
return completion, nil
|
||||
}
|
||||
if err := s.host.DisableOpenAIResponsesAPI(ctx, accountIDs); err != nil {
|
||||
return completion, nil
|
||||
}
|
||||
return s.checkGatewayCompletionWithRetry(ctx, completionReq)
|
||||
}
|
||||
|
||||
func normalizedAccountIDs(accountIDs []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
values := make([]string, 0, len(accountIDs))
|
||||
for _, rawID := range accountIDs {
|
||||
accountID := strings.TrimSpace(rawID)
|
||||
if accountID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[accountID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[accountID] = struct{}{}
|
||||
values = append(values, accountID)
|
||||
}
|
||||
return values
|
||||
}
|
||||
18
internal/access/planner.go
Normal file
18
internal/access/planner.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type closurePlan struct {
|
||||
probeAPIKey string
|
||||
}
|
||||
|
||||
func (s *Service) prepareClosurePlan(ctx context.Context, req ClosureRequest) (closurePlan, error) {
|
||||
plan := closurePlan{probeAPIKey: strings.TrimSpace(req.ProbeAPIKey)}
|
||||
if strings.TrimSpace(req.Mode) != ModeSubscription {
|
||||
return plan, nil
|
||||
}
|
||||
return s.prepareSubscriptionPlan(ctx, req, plan)
|
||||
}
|
||||
7
internal/access/self_service.go
Normal file
7
internal/access/self_service.go
Normal file
@@ -0,0 +1,7 @@
|
||||
package access
|
||||
|
||||
import "strings"
|
||||
|
||||
func selfServiceProbeAPIKey(req ClosureRequest) string {
|
||||
return strings.TrimSpace(req.ProbeAPIKey)
|
||||
}
|
||||
23
internal/access/service.go
Normal file
23
internal/access/service.go
Normal file
@@ -0,0 +1,23 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
)
|
||||
|
||||
func (s *Service) Close(ctx context.Context, req ClosureRequest) (sub2api.GatewayAccessResult, error) {
|
||||
if s == nil || s.host == nil {
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("access host is required")
|
||||
}
|
||||
if err := Validate(req); err != nil {
|
||||
return sub2api.GatewayAccessResult{}, err
|
||||
}
|
||||
|
||||
plan, err := s.prepareClosurePlan(ctx, req)
|
||||
if err != nil {
|
||||
return sub2api.GatewayAccessResult{}, err
|
||||
}
|
||||
return s.verifyGatewayClosure(ctx, req, plan)
|
||||
}
|
||||
36
internal/access/subscription.go
Normal file
36
internal/access/subscription.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
)
|
||||
|
||||
func (s *Service) prepareSubscriptionPlan(ctx context.Context, req ClosureRequest, plan closurePlan) (closurePlan, error) {
|
||||
for _, target := range req.Subscriptions {
|
||||
resolvedTarget := target.UserID
|
||||
accessRef, err := s.host.EnsureSubscriptionAccess(ctx, sub2api.EnsureSubscriptionAccessRequest{
|
||||
UserSelector: target.UserID,
|
||||
GroupID: req.GroupID,
|
||||
})
|
||||
if err != nil {
|
||||
return closurePlan{}, fmt.Errorf("ensure subscription access for %s: %w", target.UserID, err)
|
||||
}
|
||||
if strings.TrimSpace(accessRef.UserID) != "" {
|
||||
resolvedTarget = accessRef.UserID
|
||||
}
|
||||
if strings.TrimSpace(accessRef.APIKey) != "" {
|
||||
plan.probeAPIKey = strings.TrimSpace(accessRef.APIKey)
|
||||
}
|
||||
if _, err := s.host.AssignSubscription(ctx, sub2api.AssignSubscriptionRequest{
|
||||
UserID: resolvedTarget,
|
||||
GroupID: req.GroupID,
|
||||
DurationDays: target.DurationDays,
|
||||
}); err != nil {
|
||||
return closurePlan{}, fmt.Errorf("assign subscription for %s: %w", target.UserID, err)
|
||||
}
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
49
internal/access/types.go
Normal file
49
internal/access/types.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
)
|
||||
|
||||
const (
|
||||
ModeSubscription = "subscription"
|
||||
ModeSelfService = "self_service"
|
||||
|
||||
gatewayCompletionRetryAttempts = 3
|
||||
gatewayCompletionRetryDelay = 300 * time.Millisecond
|
||||
)
|
||||
|
||||
type SubscriptionTarget struct {
|
||||
UserID string
|
||||
DurationDays int
|
||||
}
|
||||
|
||||
type ClosureRequest struct {
|
||||
Mode string
|
||||
ProbeAPIKey string
|
||||
Subscriptions []SubscriptionTarget
|
||||
GroupID string
|
||||
AccountIDs []string
|
||||
ExpectedModel string
|
||||
Prompt string
|
||||
MaxTokens int
|
||||
ResponsesCapabilitySuspect bool
|
||||
}
|
||||
|
||||
type Host interface {
|
||||
EnsureSubscriptionAccess(ctx context.Context, req sub2api.EnsureSubscriptionAccessRequest) (sub2api.SubscriptionAccessRef, error)
|
||||
AssignSubscription(ctx context.Context, req sub2api.AssignSubscriptionRequest) (sub2api.SubscriptionRef, error)
|
||||
CheckGatewayAccess(ctx context.Context, req sub2api.GatewayAccessCheckRequest) (sub2api.GatewayAccessResult, error)
|
||||
CheckGatewayCompletion(ctx context.Context, req sub2api.GatewayCompletionCheckRequest) (sub2api.GatewayCompletionResult, error)
|
||||
DisableOpenAIResponsesAPI(ctx context.Context, accountIDs []string) error
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
host Host
|
||||
}
|
||||
|
||||
func NewService(host Host) *Service {
|
||||
return &Service{host: host}
|
||||
}
|
||||
25
internal/access/validation.go
Normal file
25
internal/access/validation.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func Validate(req ClosureRequest) error {
|
||||
switch strings.TrimSpace(req.Mode) {
|
||||
case ModeSubscription:
|
||||
if len(req.Subscriptions) == 0 {
|
||||
return fmt.Errorf("subscription access requires at least one subscription target")
|
||||
}
|
||||
case ModeSelfService:
|
||||
if selfServiceProbeAPIKey(req) == "" {
|
||||
return fmt.Errorf("self_service access requires probe api key")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported access mode %q", req.Mode)
|
||||
}
|
||||
if strings.TrimSpace(req.Mode) != ModeSubscription && selfServiceProbeAPIKey(req) == "" {
|
||||
return fmt.Errorf("access probe api key is required to verify gateway closure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
"sub2api-cn-relay-manager/internal/pack"
|
||||
"sub2api-cn-relay-manager/internal/provision"
|
||||
"sub2api-cn-relay-manager/internal/reconcile"
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
)
|
||||
|
||||
@@ -303,11 +304,11 @@ func TestAPIRollbackProviderReturnsSummary(t *testing.T) {
|
||||
|
||||
func TestAPIReconcileProviderReturnsSummary(t *testing.T) {
|
||||
handler := NewAPIHandler("secret-token", ActionSet{
|
||||
ReconcileProvider: func(_ context.Context, req ReconcileProviderRequest) (provision.ReconcileResult, error) {
|
||||
ReconcileProvider: func(_ context.Context, req ReconcileProviderRequest) (reconcile.Result, error) {
|
||||
if req.AccessAPIKey != "user-key" {
|
||||
t.Fatalf("AccessAPIKey = %q, want user-key", req.AccessAPIKey)
|
||||
}
|
||||
return provision.ReconcileResult{BatchID: 7, Status: "drifted", MissingCount: 1, ExtraCount: 2, StaleNoiseCount: 3, ProbeFailureCount: 1, AccessStatus: provision.AccessStatusBroken, Summary: map[string]any{"probe_failures": 1, "stale_noise_count": 3}}, nil
|
||||
return reconcile.Result{BatchID: 7, Status: "drifted", MissingCount: 1, ExtraCount: 2, StaleNoiseCount: 3, ProbeFailureCount: 1, AccessStatus: provision.AccessStatusBroken, Summary: map[string]any{"probe_failures": 1, "stale_noise_count": 3}}, nil
|
||||
},
|
||||
})
|
||||
request := httptestRequest(t, http.MethodPost, "/api/providers/deepseek/reconcile", map[string]any{"host_base_url": "https://sub2api.example.com", "pack_path": "/tmp/openai-pack.zip", "access_api_key": "user-key"}, "secret-token")
|
||||
@@ -610,9 +611,9 @@ func TestAPIRollbackProviderReturnsConfiguredError(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAPIReconcileProviderRejectsTrailingNonObjectPayload(t *testing.T) {
|
||||
handler := NewAPIHandler("secret-token", ActionSet{ReconcileProvider: func(context.Context, ReconcileProviderRequest) (provision.ReconcileResult, error) {
|
||||
handler := NewAPIHandler("secret-token", ActionSet{ReconcileProvider: func(context.Context, ReconcileProviderRequest) (reconcile.Result, error) {
|
||||
t.Fatal("ReconcileProvider should not be called for invalid JSON")
|
||||
return provision.ReconcileResult{}, nil
|
||||
return reconcile.Result{}, nil
|
||||
}})
|
||||
request, err := http.NewRequest(http.MethodPost, "/api/providers/deepseek/reconcile", strings.NewReader(`{"host_base_url":"https://sub2api.example.com"}[]`))
|
||||
if err != nil {
|
||||
@@ -789,8 +790,8 @@ func TestHandlerErrorPaths(t *testing.T) {
|
||||
path: "/api/providers/x/reconcile",
|
||||
body: `{}`,
|
||||
actionSet: ActionSet{
|
||||
ReconcileProvider: func(context.Context, ReconcileProviderRequest) (provision.ReconcileResult, error) {
|
||||
return provision.ReconcileResult{}, errors.New("boom")
|
||||
ReconcileProvider: func(context.Context, ReconcileProviderRequest) (reconcile.Result, error) {
|
||||
return reconcile.Result{}, errors.New("boom")
|
||||
},
|
||||
},
|
||||
wantStatus: http.StatusInternalServerError,
|
||||
|
||||
@@ -55,11 +55,14 @@ func (r batchImportRuntimeRunner) execute(ctx context.Context) (BatchImportRunCr
|
||||
}
|
||||
|
||||
if _, err := service.StartRun(ctx, batch.BatchImportRunRequest{
|
||||
RunID: runID,
|
||||
Mode: r.request.Mode,
|
||||
AccessMode: r.request.AccessMode,
|
||||
HostID: r.hostRow.HostID,
|
||||
Entries: entries,
|
||||
RunID: runID,
|
||||
Mode: r.request.Mode,
|
||||
AccessMode: r.request.AccessMode,
|
||||
HostID: r.hostRow.HostID,
|
||||
SubscriptionUsers: append([]string(nil), r.request.SubscriptionUsers...),
|
||||
SubscriptionDays: r.request.SubscriptionDays,
|
||||
ProbeAPIKey: r.request.ProbeAPIKey,
|
||||
Entries: entries,
|
||||
}); err != nil {
|
||||
return BatchImportRunCreateResponse{}, err
|
||||
}
|
||||
@@ -89,7 +92,14 @@ func (r batchImportRuntimeRunner) advanceRun(ctx context.Context, runID string)
|
||||
if timeout <= 0 {
|
||||
timeout = time.Second
|
||||
}
|
||||
deadline := time.Now().Add(timeout)
|
||||
return r.driveRun(ctx, runID, timeout)
|
||||
}
|
||||
|
||||
func (r batchImportRuntimeRunner) driveRun(ctx context.Context, runID string, waitBudget time.Duration) error {
|
||||
deadline := time.Now()
|
||||
if waitBudget > 0 {
|
||||
deadline = deadline.Add(waitBudget)
|
||||
}
|
||||
|
||||
worker := batch.ConfirmationWorker{
|
||||
WorkerID: "batch-import-api",
|
||||
@@ -137,7 +147,7 @@ func (r batchImportRuntimeRunner) advanceRun(ctx context.Context, runID string)
|
||||
if run.TotalItems > 0 && run.CompletedItems >= run.TotalItems {
|
||||
return nil
|
||||
}
|
||||
if !pendingWork || !time.Now().Before(deadline) {
|
||||
if !pendingWork || waitBudget <= 0 || !time.Now().Before(deadline) {
|
||||
return nil
|
||||
}
|
||||
if err := sleepWithContext(ctx, batchImportRetryDelay); err != nil {
|
||||
@@ -274,6 +284,17 @@ func (s batchImportRunItemStore) Upsert(ctx context.Context, item sqlite.ImportR
|
||||
return s.store.ImportRunItems().Upsert(ctx, item)
|
||||
}
|
||||
|
||||
func (s batchImportRunItemStore) TryAcquireLease(ctx context.Context, itemID, workerID string, now time.Time, leaseDuration time.Duration) (sqlite.ImportRunItem, bool, error) {
|
||||
item, claimed, err := s.store.ImportRunItems().TryAcquireConfirmationLease(ctx, itemID, workerID, now, leaseDuration)
|
||||
if err != nil || !claimed {
|
||||
return item, claimed, err
|
||||
}
|
||||
if item.RunID != s.runID {
|
||||
return sqlite.ImportRunItem{}, false, nil
|
||||
}
|
||||
return item, true, nil
|
||||
}
|
||||
|
||||
func generatedBatchImportPack(providerManifest pack.ProviderManifest) pack.LoadedPack {
|
||||
return pack.LoadedPack{
|
||||
Manifest: pack.Manifest{
|
||||
|
||||
88
internal/app/batch_runtime_background.go
Normal file
88
internal/app/batch_runtime_background.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
"sub2api-cn-relay-manager/internal/worker"
|
||||
)
|
||||
|
||||
const batchImportBackgroundPollInterval = time.Second
|
||||
|
||||
func runBatchImportBackgroundScheduler(ctx context.Context, sqliteDSN string) {
|
||||
worker.NewRunner(
|
||||
[]worker.Job{batchImportResumeJob{sqliteDSN: sqliteDSN}},
|
||||
batchImportBackgroundPollInterval,
|
||||
log.Printf,
|
||||
).Start(ctx)
|
||||
}
|
||||
|
||||
type batchImportResumeJob struct {
|
||||
sqliteDSN string
|
||||
}
|
||||
|
||||
func (j batchImportResumeJob) Name() string {
|
||||
return "batch import runtime scheduler"
|
||||
}
|
||||
|
||||
func (j batchImportResumeJob) Run(ctx context.Context) error {
|
||||
return resumePendingBatchImportRuns(ctx, j.sqliteDSN)
|
||||
}
|
||||
|
||||
func resumePendingBatchImportRuns(ctx context.Context, sqliteDSN string) error {
|
||||
store, err := sqlite.Open(ctx, sqliteDSN)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
runs, err := store.ImportRuns().List(ctx, 1000)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, run := range runs {
|
||||
if strings.TrimSpace(run.State) != "running" {
|
||||
continue
|
||||
}
|
||||
runner, err := newBatchImportRuntimeRunnerFromStoredRun(ctx, store, run)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runner.driveRun(ctx, run.RunID, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newBatchImportRuntimeRunnerFromStoredRun(ctx context.Context, store *sqlite.DB, run sqlite.ImportRun) (batchImportRuntimeRunner, error) {
|
||||
hostRow, client, err := resolveManagedHost(ctx, store, run.HostID, "", CreateHostAuth{})
|
||||
if err != nil {
|
||||
return batchImportRuntimeRunner{}, err
|
||||
}
|
||||
return batchImportRuntimeRunner{
|
||||
store: store,
|
||||
hostRow: hostRow,
|
||||
hostClient: client,
|
||||
request: CreateBatchImportRunRequest{
|
||||
HostID: run.HostID,
|
||||
Mode: run.Mode,
|
||||
AccessMode: run.AccessMode,
|
||||
SubscriptionUsers: parseJSONStringList(run.SubscriptionUsersJSON),
|
||||
SubscriptionDays: run.SubscriptionDays,
|
||||
ProbeAPIKey: run.ProbeAPIKey,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseJSONStringList(raw string) []string {
|
||||
values := []string{}
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(raw)), &values); err != nil {
|
||||
return []string{}
|
||||
}
|
||||
return values
|
||||
}
|
||||
111
internal/app/batch_runtime_background_test.go
Normal file
111
internal/app/batch_runtime_background_test.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
)
|
||||
|
||||
func TestResumePendingBatchImportRunsCompletesStoredRun(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(newBatchImportActionStubServer(t))
|
||||
defer server.Close()
|
||||
|
||||
dsn := fmt.Sprintf("file:%s?_busy_timeout=5000&_pragma=foreign_keys(0)", filepath.ToSlash(filepath.Join(t.TempDir(), "state.db")))
|
||||
store, err := sqlite.Open(context.Background(), dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("sqlite.Open() error = %v", err)
|
||||
}
|
||||
defer closeAppTestStore(t, store)
|
||||
if _, err := store.SQLDB().Exec("PRAGMA foreign_keys = OFF"); err != nil {
|
||||
t.Fatalf("disable foreign keys pragma error = %v", err)
|
||||
}
|
||||
|
||||
hostPK, err := store.Hosts().Create(context.Background(), sqlite.Host{
|
||||
HostID: "host-1",
|
||||
BaseURL: server.URL,
|
||||
HostVersion: "0.1.126",
|
||||
CapabilityProbeJSON: "{}",
|
||||
AuthType: "apikey",
|
||||
AuthToken: "host-token",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Hosts().Create() error = %v", err)
|
||||
}
|
||||
|
||||
if err := store.ImportRuns().Create(context.Background(), sqlite.ImportRun{
|
||||
RunID: "run-resume-1",
|
||||
HostID: "host-1",
|
||||
Mode: "strict",
|
||||
AccessMode: "self_service",
|
||||
SubscriptionUsersJSON: `[]`,
|
||||
SubscriptionDays: 0,
|
||||
ProbeAPIKey: "gateway-key",
|
||||
State: "running",
|
||||
TotalItems: 1,
|
||||
}); err != nil {
|
||||
t.Fatalf("ImportRuns().Create() error = %v", err)
|
||||
}
|
||||
|
||||
legacyBatchID := int64(81)
|
||||
if err := store.ImportRunItems().Upsert(context.Background(), sqlite.ImportRunItem{
|
||||
ItemID: "item-resume-1",
|
||||
RunID: "run-resume-1",
|
||||
BaseURL: server.URL,
|
||||
ProviderID: "provider-resume-1",
|
||||
APIKeyFingerprint: "sha256:test",
|
||||
RequestedModelsJSON: `["kimi-k2.6"]`,
|
||||
RawModelsJSON: `["kimi-k2.6"]`,
|
||||
NormalizedModelsJSON: `["kimi-k2.6"]`,
|
||||
CanonicalFamiliesJSON: `["kimi-2.6"]`,
|
||||
RecommendedModelsJSON: `["kimi-k2.6"]`,
|
||||
ResolvedSmokeModel: "kimi-k2.6",
|
||||
CapabilityProfileJSON: `{"transport_profile":{"supports_openai_chat_completions":true}}`,
|
||||
CurrentStage: "confirm",
|
||||
ConfirmationStatus: "pending",
|
||||
AccessStatus: "unknown",
|
||||
MatchedAccountState: "none",
|
||||
AccountResolution: "created",
|
||||
LegacyBatchID: &legacyBatchID,
|
||||
}); err != nil {
|
||||
t.Fatalf("ImportRunItems().Upsert() error = %v", err)
|
||||
}
|
||||
|
||||
if _, err := store.ManagedResources().Create(context.Background(), sqlite.ManagedResource{
|
||||
BatchID: legacyBatchID,
|
||||
HostID: hostPK,
|
||||
ResourceType: "account",
|
||||
HostResourceID: "account_1",
|
||||
ResourceName: "batch-import-account",
|
||||
}); err != nil {
|
||||
t.Fatalf("ManagedResources().Create(account) error = %v", err)
|
||||
}
|
||||
|
||||
if err := resumePendingBatchImportRuns(context.Background(), dsn); err != nil {
|
||||
t.Fatalf("resumePendingBatchImportRuns() error = %v", err)
|
||||
}
|
||||
|
||||
run, err := store.ImportRuns().GetByRunID(context.Background(), "run-resume-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ImportRuns().GetByRunID() error = %v", err)
|
||||
}
|
||||
if run.State != "completed" {
|
||||
t.Fatalf("run.State = %q, want completed", run.State)
|
||||
}
|
||||
if run.CompletedItems != 1 || run.ActiveItems != 1 {
|
||||
t.Fatalf("run counters = %+v, want completed_items=1 active_items=1", run)
|
||||
}
|
||||
|
||||
item, err := store.ImportRunItems().GetByItemID(context.Background(), "item-resume-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ImportRunItems().GetByItemID() error = %v", err)
|
||||
}
|
||||
if item.CurrentStage != "done" || item.ConfirmationStatus != "confirmed" || item.AccessStatus != "active" {
|
||||
t.Fatalf("item = %+v, want done/confirmed/active", item)
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,12 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/config"
|
||||
)
|
||||
|
||||
func Bootstrap(_ context.Context) (*Server, error) {
|
||||
func Bootstrap(ctx context.Context) (*Server, error) {
|
||||
cfg, err := config.LoadStartupFromEnv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -15,6 +16,28 @@ func Bootstrap(_ context.Context) (*Server, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
startBackgroundSchedulers(ctx, cfg, defaultBackgroundSchedulers())
|
||||
handler := NewAPIHandler(adminToken, NewActionSet(cfg.Database.SQLiteDSN))
|
||||
return NewServer(cfg.Server.ListenAddr, handler, nil), nil
|
||||
}
|
||||
|
||||
type backgroundSchedulers struct {
|
||||
runBatchImport func(context.Context, string)
|
||||
runReconcile func(context.Context, string, time.Duration)
|
||||
}
|
||||
|
||||
func defaultBackgroundSchedulers() backgroundSchedulers {
|
||||
return backgroundSchedulers{
|
||||
runBatchImport: runBatchImportBackgroundScheduler,
|
||||
runReconcile: runReconcileBackgroundScheduler,
|
||||
}
|
||||
}
|
||||
|
||||
func startBackgroundSchedulers(ctx context.Context, cfg config.StartupConfig, schedulers backgroundSchedulers) {
|
||||
if schedulers.runBatchImport != nil {
|
||||
schedulers.runBatchImport(ctx, cfg.Database.SQLiteDSN)
|
||||
}
|
||||
if cfg.Reconcile.WorkerEnabled && schedulers.runReconcile != nil {
|
||||
schedulers.runReconcile(ctx, cfg.Database.SQLiteDSN, cfg.Reconcile.PollInterval)
|
||||
}
|
||||
}
|
||||
|
||||
59
internal/app/bootstrap_test.go
Normal file
59
internal/app/bootstrap_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/config"
|
||||
)
|
||||
|
||||
func TestStartBackgroundSchedulersHonorsReconcileWorkerFlag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
batchCalls := 0
|
||||
reconcileCalls := 0
|
||||
startBackgroundSchedulers(context.Background(), config.StartupConfig{
|
||||
Database: config.DatabaseConfig{SQLiteDSN: "file:test.db"},
|
||||
Reconcile: config.ReconcileConfig{
|
||||
WorkerEnabled: true,
|
||||
PollInterval: 15 * time.Minute,
|
||||
},
|
||||
}, backgroundSchedulers{
|
||||
runBatchImport: func(context.Context, string) {
|
||||
batchCalls++
|
||||
},
|
||||
runReconcile: func(context.Context, string, time.Duration) {
|
||||
reconcileCalls++
|
||||
},
|
||||
})
|
||||
|
||||
if batchCalls != 1 {
|
||||
t.Fatalf("batchCalls = %d, want 1", batchCalls)
|
||||
}
|
||||
if reconcileCalls != 1 {
|
||||
t.Fatalf("reconcileCalls = %d, want 1", reconcileCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartBackgroundSchedulersSkipsDisabledReconcileWorker(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
reconcileCalls := 0
|
||||
startBackgroundSchedulers(context.Background(), config.StartupConfig{
|
||||
Database: config.DatabaseConfig{SQLiteDSN: "file:test.db"},
|
||||
Reconcile: config.ReconcileConfig{
|
||||
WorkerEnabled: false,
|
||||
PollInterval: 15 * time.Minute,
|
||||
},
|
||||
}, backgroundSchedulers{
|
||||
runBatchImport: func(context.Context, string) {},
|
||||
runReconcile: func(context.Context, string, time.Duration) {
|
||||
reconcileCalls++
|
||||
},
|
||||
})
|
||||
|
||||
if reconcileCalls != 0 {
|
||||
t.Fatalf("reconcileCalls = %d, want 0", reconcileCalls)
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
"sub2api-cn-relay-manager/internal/pack"
|
||||
"sub2api-cn-relay-manager/internal/provision"
|
||||
"sub2api-cn-relay-manager/internal/reconcile"
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/access"
|
||||
@@ -21,9 +22,9 @@ import (
|
||||
|
||||
type ActionSet struct {
|
||||
CreateBatchImportRun func(context.Context, CreateBatchImportRunRequest) (BatchImportRunCreateResponse, error)
|
||||
ListBatchImportRuns func(context.Context, ListBatchImportRunsRequest) ([]batch.RunSummaryProjection, error)
|
||||
ListBatchImportRuns func(context.Context, ListBatchImportRunsRequest) (ListBatchImportRunsResponse, error)
|
||||
GetBatchImportRun func(context.Context, string) (batch.RunSummaryProjection, error)
|
||||
ListBatchImportRunItems func(context.Context, ListBatchImportRunItemsRequest) ([]batch.ItemSummaryProjection, error)
|
||||
ListBatchImportRunItems func(context.Context, ListBatchImportRunItemsRequest) (ListBatchImportRunItemsResponse, error)
|
||||
GetBatchImportRunItem func(context.Context, GetBatchImportRunItemRequest) (batch.ItemDetailProjection, error)
|
||||
InstallPack func(context.Context, InstallPackRequest) (provision.PackInstallResult, error)
|
||||
BatchDetail func(context.Context, BatchDetailRequest) (provision.BatchDetailResult, error)
|
||||
@@ -35,7 +36,7 @@ type ActionSet struct {
|
||||
ImportProvider func(context.Context, ImportProviderRequest) (provision.RuntimeImportResult, error)
|
||||
RollbackProvider func(context.Context, RollbackProviderRequest) (provision.RollbackReport, error)
|
||||
RollbackBatch func(context.Context, RollbackBatchRequest) (provision.RollbackReport, error)
|
||||
ReconcileProvider func(context.Context, ReconcileProviderRequest) (provision.ReconcileResult, error)
|
||||
ReconcileProvider func(context.Context, ReconcileProviderRequest) (reconcile.Result, error)
|
||||
CreateHost func(context.Context, CreateHostRequest) (HostInfo, error)
|
||||
ProbeHost func(context.Context, ProbeHostRequest) (HostInfo, error)
|
||||
ListHosts func(context.Context) ([]HostInfo, error)
|
||||
@@ -713,7 +714,7 @@ func handleRollbackBatch(w http.ResponseWriter, r *http.Request, fn func(context
|
||||
})
|
||||
}
|
||||
|
||||
func handleReconcileProvider(w http.ResponseWriter, r *http.Request, fn func(context.Context, ReconcileProviderRequest) (provision.ReconcileResult, error)) {
|
||||
func handleReconcileProvider(w http.ResponseWriter, r *http.Request, fn func(context.Context, ReconcileProviderRequest) (reconcile.Result, error)) {
|
||||
if fn == nil {
|
||||
writeHTTPError(w, &httpError{StatusCode: http.StatusInternalServerError, Code: "server_misconfigured", Message: "reconcile-provider action is not configured"})
|
||||
return
|
||||
@@ -1124,26 +1125,26 @@ func NewActionSet(sqliteDSN string) ActionSet {
|
||||
}
|
||||
return report, nil
|
||||
},
|
||||
ReconcileProvider: func(ctx context.Context, req ReconcileProviderRequest) (provision.ReconcileResult, error) {
|
||||
ReconcileProvider: func(ctx context.Context, req ReconcileProviderRequest) (reconcile.Result, error) {
|
||||
loadedPack, err := pack.LoadPath(req.PackPath)
|
||||
if err != nil {
|
||||
return provision.ReconcileResult{}, err
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
providerManifest, err := findProvider(loadedPack, req.ProviderID)
|
||||
if err != nil {
|
||||
return provision.ReconcileResult{}, err
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
store, err := sqlite.Open(ctx, sqliteDSN)
|
||||
if err != nil {
|
||||
return provision.ReconcileResult{}, err
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
defer store.Close()
|
||||
hostRow, client, err := resolveManagedHost(ctx, store, req.HostID, req.HostBaseURL, createHostAuthFromLegacyFields(req.HostAPIKey, req.HostBearerToken))
|
||||
if err != nil {
|
||||
return provision.ReconcileResult{}, err
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
service := provision.NewReconcileService(store, client)
|
||||
return service.Reconcile(ctx, provision.ReconcileRequest{HostID: hostRow.HostID, HostBaseURL: hostRow.BaseURL, AccessProbeAPIKey: req.AccessAPIKey, Pack: loadedPack, Provider: providerManifest})
|
||||
service := reconcile.NewService(store, client)
|
||||
return service.Reconcile(ctx, reconcile.Request{HostID: hostRow.HostID, HostBaseURL: hostRow.BaseURL, AccessProbeAPIKey: req.AccessAPIKey, Pack: loadedPack, Provider: providerManifest})
|
||||
},
|
||||
CreateHost: func(ctx context.Context, req CreateHostRequest) (HostInfo, error) {
|
||||
if strings.TrimSpace(req.BaseURL) == "" {
|
||||
|
||||
@@ -42,9 +42,15 @@ type ListBatchImportRunsRequest struct {
|
||||
State string
|
||||
AccessMode string
|
||||
Query string
|
||||
Cursor string
|
||||
Limit int
|
||||
}
|
||||
|
||||
type ListBatchImportRunsResponse struct {
|
||||
Runs []batch.RunSummaryProjection `json:"runs"`
|
||||
NextCursor *string `json:"next_cursor"`
|
||||
}
|
||||
|
||||
type ListBatchImportRunItemsRequest struct {
|
||||
RunID string
|
||||
CurrentStage string
|
||||
@@ -55,9 +61,15 @@ type ListBatchImportRunItemsRequest struct {
|
||||
MatchedAccountState string
|
||||
AccountResolution string
|
||||
Query string
|
||||
Cursor string
|
||||
Limit int
|
||||
}
|
||||
|
||||
type ListBatchImportRunItemsResponse struct {
|
||||
Items []batch.ItemSummaryProjection `json:"items"`
|
||||
NextCursor *string `json:"next_cursor"`
|
||||
}
|
||||
|
||||
type GetBatchImportRunItemRequest struct {
|
||||
RunID string
|
||||
ItemID string
|
||||
@@ -85,7 +97,7 @@ func handleCreateBatchImportRun(w http.ResponseWriter, r *http.Request, fn func(
|
||||
writeJSON(w, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func handleListBatchImportRuns(w http.ResponseWriter, r *http.Request, fn func(context.Context, ListBatchImportRunsRequest) ([]batch.RunSummaryProjection, error)) {
|
||||
func handleListBatchImportRuns(w http.ResponseWriter, r *http.Request, fn func(context.Context, ListBatchImportRunsRequest) (ListBatchImportRunsResponse, error)) {
|
||||
if fn == nil {
|
||||
writeHTTPError(w, &httpError{StatusCode: http.StatusInternalServerError, Code: "server_misconfigured", Message: "list-batch-import-runs action is not configured"})
|
||||
return
|
||||
@@ -94,16 +106,17 @@ func handleListBatchImportRuns(w http.ResponseWriter, r *http.Request, fn func(c
|
||||
State: strings.TrimSpace(r.URL.Query().Get("state")),
|
||||
AccessMode: strings.TrimSpace(r.URL.Query().Get("access_mode")),
|
||||
Query: strings.TrimSpace(r.URL.Query().Get("q")),
|
||||
Cursor: strings.TrimSpace(r.URL.Query().Get("cursor")),
|
||||
Limit: parsePositiveInt(r.URL.Query().Get("limit")),
|
||||
})
|
||||
if err != nil {
|
||||
writeHTTPError(w, classifyError(err))
|
||||
return
|
||||
}
|
||||
if result == nil {
|
||||
result = []batch.RunSummaryProjection{}
|
||||
if result.Runs == nil {
|
||||
result.Runs = []batch.RunSummaryProjection{}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"runs": result})
|
||||
writeJSON(w, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func buildCreateBatchImportRunAction(sqliteDSN string) func(context.Context, CreateBatchImportRunRequest) (BatchImportRunCreateResponse, error) {
|
||||
@@ -128,33 +141,63 @@ func buildCreateBatchImportRunAction(sqliteDSN string) func(context.Context, Cre
|
||||
}
|
||||
}
|
||||
|
||||
func buildListBatchImportRunsAction(sqliteDSN string) func(context.Context, ListBatchImportRunsRequest) ([]batch.RunSummaryProjection, error) {
|
||||
return func(ctx context.Context, req ListBatchImportRunsRequest) ([]batch.RunSummaryProjection, error) {
|
||||
func buildListBatchImportRunsAction(sqliteDSN string) func(context.Context, ListBatchImportRunsRequest) (ListBatchImportRunsResponse, error) {
|
||||
return func(ctx context.Context, req ListBatchImportRunsRequest) (ListBatchImportRunsResponse, error) {
|
||||
store, err := sqlite.Open(ctx, sqliteDSN)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return ListBatchImportRunsResponse{}, err
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
runs, err := store.ImportRuns().List(ctx, defaultPositiveInt(req.Limit, 50))
|
||||
runs, err := store.ImportRuns().List(ctx, 1000)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return ListBatchImportRunsResponse{}, err
|
||||
}
|
||||
|
||||
result := make([]batch.RunSummaryProjection, 0, len(runs))
|
||||
limit := defaultPositiveInt(req.Limit, 50)
|
||||
result := make([]batch.RunSummaryProjection, 0, limit)
|
||||
nextCursor := (*string)(nil)
|
||||
started := strings.TrimSpace(req.Cursor) == ""
|
||||
for _, run := range runs {
|
||||
if !started {
|
||||
if run.RunID == strings.TrimSpace(req.Cursor) {
|
||||
started = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if req.State != "" && run.State != req.State {
|
||||
continue
|
||||
}
|
||||
if req.AccessMode != "" && run.AccessMode != req.AccessMode {
|
||||
continue
|
||||
}
|
||||
if req.Query != "" && !strings.Contains(strings.ToLower(run.RunID), strings.ToLower(req.Query)) {
|
||||
continue
|
||||
if req.Query != "" {
|
||||
query := strings.ToLower(req.Query)
|
||||
if !strings.Contains(strings.ToLower(run.RunID), query) {
|
||||
items, err := store.ImportRunItems().ListByRunID(ctx, run.RunID)
|
||||
if err != nil {
|
||||
return ListBatchImportRunsResponse{}, err
|
||||
}
|
||||
matched := false
|
||||
for _, item := range items {
|
||||
if strings.Contains(strings.ToLower(item.ProviderID), query) || strings.Contains(strings.ToLower(item.BaseURL), query) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(result) >= limit {
|
||||
cursor := run.RunID
|
||||
nextCursor = &cursor
|
||||
break
|
||||
}
|
||||
result = append(result, batch.ProjectRunSummary(run))
|
||||
}
|
||||
return result, nil
|
||||
return ListBatchImportRunsResponse{Runs: result, NextCursor: nextCursor}, nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ func handleGetBatchImportRun(w http.ResponseWriter, r *http.Request, fn func(con
|
||||
})
|
||||
}
|
||||
|
||||
func handleListBatchImportRunItems(w http.ResponseWriter, r *http.Request, fn func(context.Context, ListBatchImportRunItemsRequest) ([]batch.ItemSummaryProjection, error)) {
|
||||
func handleListBatchImportRunItems(w http.ResponseWriter, r *http.Request, fn func(context.Context, ListBatchImportRunItemsRequest) (ListBatchImportRunItemsResponse, error)) {
|
||||
if fn == nil {
|
||||
writeHTTPError(w, &httpError{StatusCode: http.StatusInternalServerError, Code: "server_misconfigured", Message: "list-batch-import-run-items action is not configured"})
|
||||
return
|
||||
@@ -51,6 +51,7 @@ func handleListBatchImportRunItems(w http.ResponseWriter, r *http.Request, fn fu
|
||||
MatchedAccountState: strings.TrimSpace(r.URL.Query().Get("matched_account_state")),
|
||||
AccountResolution: strings.TrimSpace(r.URL.Query().Get("account_resolution")),
|
||||
Query: strings.TrimSpace(r.URL.Query().Get("q")),
|
||||
Cursor: strings.TrimSpace(r.URL.Query().Get("cursor")),
|
||||
Limit: parsePositiveInt(r.URL.Query().Get("limit")),
|
||||
}
|
||||
if hasWarningRaw := strings.TrimSpace(r.URL.Query().Get("has_warning")); hasWarningRaw != "" {
|
||||
@@ -62,10 +63,10 @@ func handleListBatchImportRunItems(w http.ResponseWriter, r *http.Request, fn fu
|
||||
writeHTTPError(w, classifyError(err))
|
||||
return
|
||||
}
|
||||
if items == nil {
|
||||
items = []batch.ItemSummaryProjection{}
|
||||
if items.Items == nil {
|
||||
items.Items = []batch.ItemSummaryProjection{}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"items": items})
|
||||
writeJSON(w, http.StatusOK, items)
|
||||
}
|
||||
|
||||
func handleGetBatchImportRunItem(w http.ResponseWriter, r *http.Request, fn func(context.Context, GetBatchImportRunItemRequest) (batch.ItemDetailProjection, error)) {
|
||||
@@ -108,37 +109,48 @@ func buildGetBatchImportRunAction(sqliteDSN string) func(context.Context, string
|
||||
}
|
||||
}
|
||||
|
||||
func buildListBatchImportRunItemsAction(sqliteDSN string) func(context.Context, ListBatchImportRunItemsRequest) ([]batch.ItemSummaryProjection, error) {
|
||||
return func(ctx context.Context, req ListBatchImportRunItemsRequest) ([]batch.ItemSummaryProjection, error) {
|
||||
func buildListBatchImportRunItemsAction(sqliteDSN string) func(context.Context, ListBatchImportRunItemsRequest) (ListBatchImportRunItemsResponse, error) {
|
||||
return func(ctx context.Context, req ListBatchImportRunItemsRequest) (ListBatchImportRunItemsResponse, error) {
|
||||
store, err := sqlite.Open(ctx, sqliteDSN)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return ListBatchImportRunItemsResponse{}, err
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
if _, err := store.ImportRuns().GetByRunID(ctx, req.RunID); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("run not found: %s", req.RunID)
|
||||
return ListBatchImportRunItemsResponse{}, fmt.Errorf("run not found: %s", req.RunID)
|
||||
}
|
||||
return nil, err
|
||||
return ListBatchImportRunItemsResponse{}, err
|
||||
}
|
||||
items, err := store.ImportRunItems().ListByRunID(ctx, req.RunID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return ListBatchImportRunItemsResponse{}, err
|
||||
}
|
||||
|
||||
result := make([]batch.ItemSummaryProjection, 0, len(items))
|
||||
limit := defaultPositiveInt(req.Limit, 50)
|
||||
result := make([]batch.ItemSummaryProjection, 0, limit)
|
||||
nextCursor := (*string)(nil)
|
||||
started := strings.TrimSpace(req.Cursor) == ""
|
||||
for _, item := range items {
|
||||
if !started {
|
||||
if item.ItemID == strings.TrimSpace(req.Cursor) {
|
||||
started = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
view := batch.ProjectItemSummary(item)
|
||||
if !matchesItemFilters(view, req) {
|
||||
continue
|
||||
}
|
||||
result = append(result, view)
|
||||
if req.Limit > 0 && len(result) >= req.Limit {
|
||||
if len(result) >= limit {
|
||||
cursor := item.ItemID
|
||||
nextCursor = &cursor
|
||||
break
|
||||
}
|
||||
result = append(result, view)
|
||||
}
|
||||
return result, nil
|
||||
return ListBatchImportRunItemsResponse{Items: result, NextCursor: nextCursor}, nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,25 +17,32 @@ func TestBatchRunsHTTP(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
handler := NewAPIHandler("secret-token", ActionSet{
|
||||
ListBatchImportRuns: func(_ context.Context, req ListBatchImportRunsRequest) ([]batch.RunSummaryProjection, error) {
|
||||
ListBatchImportRuns: func(_ context.Context, req ListBatchImportRunsRequest) (ListBatchImportRunsResponse, error) {
|
||||
if req.State != "completed_with_warnings" {
|
||||
t.Fatalf("State = %q, want completed_with_warnings", req.State)
|
||||
}
|
||||
if req.AccessMode != "subscription" {
|
||||
t.Fatalf("AccessMode = %q, want subscription", req.AccessMode)
|
||||
}
|
||||
return []batch.RunSummaryProjection{{
|
||||
if req.Query != "kimi" {
|
||||
t.Fatalf("Query = %q, want kimi", req.Query)
|
||||
}
|
||||
if req.Cursor != "cursor-1" {
|
||||
t.Fatalf("Cursor = %q, want cursor-1", req.Cursor)
|
||||
}
|
||||
nextCursor := "cursor-2"
|
||||
return ListBatchImportRunsResponse{Runs: []batch.RunSummaryProjection{{
|
||||
RunID: "run-1",
|
||||
State: "completed_with_warnings",
|
||||
Mode: "partial",
|
||||
AccessMode: "subscription",
|
||||
TotalItems: 2,
|
||||
WarningItems: 1,
|
||||
}}, nil
|
||||
}}, NextCursor: &nextCursor}, nil
|
||||
},
|
||||
})
|
||||
|
||||
req := httptestRequest(t, http.MethodGet, "/api/batch-import/runs?state=completed_with_warnings&access_mode=subscription", nil, "secret-token")
|
||||
req := httptestRequest(t, http.MethodGet, "/api/batch-import/runs?state=completed_with_warnings&access_mode=subscription&q=kimi&cursor=cursor-1", nil, "secret-token")
|
||||
res := httptestRecorder(handler, req)
|
||||
assertStatusCode(t, res, http.StatusOK)
|
||||
run := decodeJSONArrayObjectAt(t, res.Body().Bytes(), "runs", 0)
|
||||
@@ -43,6 +50,7 @@ func TestBatchRunsHTTP(t *testing.T) {
|
||||
assertJSONObjectValue(t, run, "state", "completed_with_warnings")
|
||||
assertJSONObjectValue(t, run, "access_mode", "subscription")
|
||||
assertJSONObjectValue(t, run, "warning_items", float64(1))
|
||||
assertJSONContains(t, res.Body().Bytes(), "next_cursor", "cursor-2")
|
||||
})
|
||||
|
||||
t.Run("GET run detail returns wrapped projection", func(t *testing.T) {
|
||||
@@ -79,7 +87,7 @@ func TestBatchRunsHTTP(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
handler := NewAPIHandler("secret-token", ActionSet{
|
||||
ListBatchImportRunItems: func(_ context.Context, req ListBatchImportRunItemsRequest) ([]batch.ItemSummaryProjection, error) {
|
||||
ListBatchImportRunItems: func(_ context.Context, req ListBatchImportRunItemsRequest) (ListBatchImportRunItemsResponse, error) {
|
||||
if req.RunID != "run-1" {
|
||||
t.Fatalf("RunID = %q, want run-1", req.RunID)
|
||||
}
|
||||
@@ -89,7 +97,11 @@ func TestBatchRunsHTTP(t *testing.T) {
|
||||
if req.AccountResolution != "reused" {
|
||||
t.Fatalf("AccountResolution = %q, want reused", req.AccountResolution)
|
||||
}
|
||||
return []batch.ItemSummaryProjection{{
|
||||
if req.Cursor != "item-cursor-1" {
|
||||
t.Fatalf("Cursor = %q, want item-cursor-1", req.Cursor)
|
||||
}
|
||||
nextCursor := "item-cursor-2"
|
||||
return ListBatchImportRunItemsResponse{Items: []batch.ItemSummaryProjection{{
|
||||
ItemID: "item-1",
|
||||
BaseURL: "https://kimi.example.com/v1",
|
||||
ProviderID: "kimi-a7m-1",
|
||||
@@ -100,11 +112,11 @@ func TestBatchRunsHTTP(t *testing.T) {
|
||||
MatchedAccountState: "active",
|
||||
AccountResolution: "reused",
|
||||
ProvisionReused: true,
|
||||
}}, nil
|
||||
}}, NextCursor: &nextCursor}, nil
|
||||
},
|
||||
})
|
||||
|
||||
req := httptestRequest(t, http.MethodGet, "/api/batch-import/runs/run-1/items?matched_account_state=active&account_resolution=reused", nil, "secret-token")
|
||||
req := httptestRequest(t, http.MethodGet, "/api/batch-import/runs/run-1/items?matched_account_state=active&account_resolution=reused&cursor=item-cursor-1", nil, "secret-token")
|
||||
res := httptestRecorder(handler, req)
|
||||
assertStatusCode(t, res, http.StatusOK)
|
||||
item := decodeJSONArrayObjectAt(t, res.Body().Bytes(), "items", 0)
|
||||
@@ -112,6 +124,7 @@ func TestBatchRunsHTTP(t *testing.T) {
|
||||
assertJSONObjectValue(t, item, "matched_account_state", "active")
|
||||
assertJSONObjectValue(t, item, "account_resolution", "reused")
|
||||
assertJSONObjectValue(t, item, "provision_reused", true)
|
||||
assertJSONContains(t, res.Body().Bytes(), "next_cursor", "item-cursor-2")
|
||||
})
|
||||
|
||||
t.Run("GET item detail returns capability profile and events", func(t *testing.T) {
|
||||
|
||||
312
internal/app/reconcile_background.go
Normal file
312
internal/app/reconcile_background.go
Normal file
@@ -0,0 +1,312 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
"sub2api-cn-relay-manager/internal/pack"
|
||||
"sub2api-cn-relay-manager/internal/provision"
|
||||
"sub2api-cn-relay-manager/internal/reconcile"
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
"sub2api-cn-relay-manager/internal/worker"
|
||||
)
|
||||
|
||||
const sqliteTimestampLayout = "2006-01-02 15:04:05"
|
||||
|
||||
func runReconcileBackgroundScheduler(ctx context.Context, sqliteDSN string, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
return
|
||||
}
|
||||
worker.NewRunner(
|
||||
[]worker.Job{reconcileSweepJob{sqliteDSN: sqliteDSN, interval: interval}},
|
||||
interval,
|
||||
log.Printf,
|
||||
).Start(ctx)
|
||||
}
|
||||
|
||||
type reconcileSweepJob struct {
|
||||
sqliteDSN string
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func (j reconcileSweepJob) Name() string {
|
||||
return "reconcile background scheduler"
|
||||
}
|
||||
|
||||
func (j reconcileSweepJob) Run(ctx context.Context) error {
|
||||
store, err := sqlite.Open(ctx, j.sqliteDSN)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer store.Close()
|
||||
return runReconcileBackgroundSweep(ctx, store, j.interval, time.Now())
|
||||
}
|
||||
|
||||
func runReconcileBackgroundSweep(ctx context.Context, store *sqlite.DB, interval time.Duration, now time.Time) error {
|
||||
if store == nil {
|
||||
return fmt.Errorf("store is required")
|
||||
}
|
||||
candidates, err := store.ImportBatches().ListLatestReconcilable(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var errs []error
|
||||
for _, batch := range candidates {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
lastRun, err := latestReconcileRunForBatch(ctx, store, batch.ProviderID, batch.HostID)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("load latest reconcile run for batch %d: %w", batch.ID, err))
|
||||
continue
|
||||
}
|
||||
if !reconcileRunDue(now, lastRun, interval) {
|
||||
continue
|
||||
}
|
||||
if err := runReconcileCandidate(ctx, store, batch); err != nil {
|
||||
errs = append(errs, fmt.Errorf("run reconcile for batch %d: %w", batch.ID, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func latestReconcileRunForBatch(ctx context.Context, store *sqlite.DB, providerID, hostID int64) (*sqlite.ReconcileRun, error) {
|
||||
runs, err := store.ReconcileRuns().GetByProviderIDAndHostID(ctx, providerID, hostID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(runs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &runs[0], nil
|
||||
}
|
||||
|
||||
func reconcileRunDue(now time.Time, run *sqlite.ReconcileRun, interval time.Duration) bool {
|
||||
if run == nil || interval <= 0 {
|
||||
return true
|
||||
}
|
||||
lastRunAt, err := time.ParseInLocation(sqliteTimestampLayout, strings.TrimSpace(run.CreatedAt), time.UTC)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
return now.Sub(lastRunAt) >= interval
|
||||
}
|
||||
|
||||
func runReconcileCandidate(ctx context.Context, store *sqlite.DB, batch sqlite.ImportBatch) error {
|
||||
hostRow, err := store.Hosts().GetByID(ctx, batch.HostID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
packRow, err := store.Packs().GetByID(ctx, batch.PackID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
providerRow, err := store.Providers().GetByID(ctx, batch.ProviderID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
accessClosures, err := store.AccessClosures().GetByBatchID(ctx, batch.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
accessProbeAPIKey, err := reconcileProbeAPIKey(ctx, store, hostRow, batch, accessClosures)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
loadedPack, err := storedLoadedPack(packRow)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
providerManifest, err := storedProviderManifest(providerRow)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client, err := newSub2APIClient(hostRow.BaseURL, authFromStoredHost(hostRow))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = reconcile.NewService(store, client).Reconcile(ctx, reconcile.Request{
|
||||
HostID: hostRow.HostID,
|
||||
HostBaseURL: hostRow.BaseURL,
|
||||
AccessProbeAPIKey: accessProbeAPIKey,
|
||||
Pack: loadedPack,
|
||||
Provider: providerManifest,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func reconcileProbeAPIKey(ctx context.Context, store *sqlite.DB, hostRow sqlite.Host, batch sqlite.ImportBatch, accessClosures []sqlite.AccessClosureRecord) (string, error) {
|
||||
if len(accessClosures) == 0 {
|
||||
return "", fmt.Errorf("access closure not found for batch %d", batch.ID)
|
||||
}
|
||||
latestClosure := accessClosures[len(accessClosures)-1]
|
||||
switch strings.TrimSpace(latestClosure.ClosureType) {
|
||||
case provision.AccessModeSelfService:
|
||||
details := parseAccessClosureDetails(latestClosure.DetailsJSON)
|
||||
apiKey, _ := details["access_api_key"].(string)
|
||||
if strings.TrimSpace(apiKey) == "" {
|
||||
apiKey, _ = details["probe_api_key"].(string)
|
||||
}
|
||||
if strings.TrimSpace(apiKey) == "" {
|
||||
return "", fmt.Errorf("self_service access closure missing probe api key")
|
||||
}
|
||||
return strings.TrimSpace(apiKey), nil
|
||||
case provision.AccessModeSubscription:
|
||||
details := parseAccessClosureDetails(latestClosure.DetailsJSON)
|
||||
subscriptionUsers := parseJSONStringArray(details["subscription_users"])
|
||||
if len(subscriptionUsers) == 0 {
|
||||
return "", fmt.Errorf("subscription access closure missing subscription_users")
|
||||
}
|
||||
subscriptionDays := parseJSONInt(details["subscription_days"])
|
||||
groupID, err := resolveManagedResourceHostIDByBatch(ctx, store, batch.ID, "group")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
client, err := newSub2APIClient(hostRow.BaseURL, authFromStoredHost(hostRow))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
accessRef, err := client.EnsureSubscriptionAccess(ctx, sub2api.EnsureSubscriptionAccessRequest{
|
||||
UserSelector: subscriptionUsers[0],
|
||||
GroupID: groupID,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
userID := strings.TrimSpace(accessRef.UserID)
|
||||
if userID == "" {
|
||||
userID = subscriptionUsers[0]
|
||||
}
|
||||
if subscriptionDays > 0 {
|
||||
if _, err := client.AssignSubscription(ctx, sub2api.AssignSubscriptionRequest{
|
||||
UserID: userID,
|
||||
GroupID: groupID,
|
||||
DurationDays: subscriptionDays,
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(accessRef.APIKey) == "" {
|
||||
return "", fmt.Errorf("subscription access api key is empty")
|
||||
}
|
||||
return strings.TrimSpace(accessRef.APIKey), nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported access closure type %q", latestClosure.ClosureType)
|
||||
}
|
||||
}
|
||||
|
||||
func parseAccessClosureDetails(raw string) map[string]any {
|
||||
payload := map[string]any{}
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(raw)), &payload); err != nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func parseJSONStringArray(raw any) []string {
|
||||
values, ok := raw.([]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
result := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
text, ok := value.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if trimmed := strings.TrimSpace(text); trimmed != "" {
|
||||
result = append(result, trimmed)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func parseJSONInt(raw any) int {
|
||||
switch value := raw.(type) {
|
||||
case float64:
|
||||
return int(value)
|
||||
case int:
|
||||
return value
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func storedLoadedPack(packRow sqlite.Pack) (pack.LoadedPack, error) {
|
||||
manifest := pack.Manifest{}
|
||||
if trimmed := strings.TrimSpace(packRow.ManifestJSON); trimmed != "" && trimmed != "{}" {
|
||||
if err := json.Unmarshal([]byte(trimmed), &manifest); err != nil {
|
||||
return pack.LoadedPack{}, fmt.Errorf("decode stored pack manifest: %w", err)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(manifest.PackID) == "" {
|
||||
manifest.PackID = strings.TrimSpace(packRow.PackID)
|
||||
}
|
||||
if strings.TrimSpace(manifest.Version) == "" {
|
||||
manifest.Version = strings.TrimSpace(packRow.Version)
|
||||
}
|
||||
if strings.TrimSpace(manifest.Vendor) == "" {
|
||||
manifest.Vendor = strings.TrimSpace(packRow.Vendor)
|
||||
}
|
||||
if strings.TrimSpace(manifest.TargetHost) == "" {
|
||||
manifest.TargetHost = strings.TrimSpace(packRow.TargetHost)
|
||||
}
|
||||
if strings.TrimSpace(manifest.MinHostVersion) == "" {
|
||||
manifest.MinHostVersion = strings.TrimSpace(packRow.MinHostVersion)
|
||||
}
|
||||
if strings.TrimSpace(manifest.MaxHostVersion) == "" {
|
||||
manifest.MaxHostVersion = strings.TrimSpace(packRow.MaxHostVersion)
|
||||
}
|
||||
return pack.LoadedPack{Manifest: manifest, Checksum: strings.TrimSpace(packRow.Checksum)}, nil
|
||||
}
|
||||
|
||||
func storedProviderManifest(providerRow sqlite.Provider) (pack.ProviderManifest, error) {
|
||||
provider := pack.ProviderManifest{}
|
||||
if trimmed := strings.TrimSpace(providerRow.ManifestJSON); trimmed != "" && trimmed != "{}" {
|
||||
if err := json.Unmarshal([]byte(trimmed), &provider); err != nil {
|
||||
return pack.ProviderManifest{}, fmt.Errorf("decode stored provider manifest: %w", err)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(provider.ProviderID) == "" {
|
||||
provider.ProviderID = strings.TrimSpace(providerRow.ProviderID)
|
||||
}
|
||||
if strings.TrimSpace(provider.DisplayName) == "" {
|
||||
provider.DisplayName = strings.TrimSpace(providerRow.DisplayName)
|
||||
}
|
||||
if strings.TrimSpace(provider.BaseURL) == "" {
|
||||
provider.BaseURL = strings.TrimSpace(providerRow.BaseURL)
|
||||
}
|
||||
if strings.TrimSpace(provider.Platform) == "" {
|
||||
provider.Platform = strings.TrimSpace(providerRow.Platform)
|
||||
}
|
||||
if strings.TrimSpace(provider.AccountType) == "" {
|
||||
provider.AccountType = strings.TrimSpace(providerRow.AccountType)
|
||||
}
|
||||
if strings.TrimSpace(provider.SmokeTestModel) == "" {
|
||||
provider.SmokeTestModel = strings.TrimSpace(providerRow.SmokeTestModel)
|
||||
}
|
||||
return provider, nil
|
||||
}
|
||||
|
||||
func resolveManagedResourceHostIDByBatch(ctx context.Context, store *sqlite.DB, batchID int64, resourceType string) (string, error) {
|
||||
resources, err := store.ManagedResources().GetByBatchID(ctx, batchID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resourceType = strings.TrimSpace(resourceType)
|
||||
for _, resource := range resources {
|
||||
if strings.TrimSpace(resource.ResourceType) == resourceType && strings.TrimSpace(resource.HostResourceID) != "" {
|
||||
return strings.TrimSpace(resource.HostResourceID), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("managed resource %q not found for batch %d", resourceType, batchID)
|
||||
}
|
||||
171
internal/app/reconcile_background_test.go
Normal file
171
internal/app/reconcile_background_test.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/pack"
|
||||
"sub2api-cn-relay-manager/internal/provision"
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
)
|
||||
|
||||
func TestRunReconcileBackgroundSweepCreatesReconcileRunForLatestSuccessfulBatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(newBatchImportActionStubServer(t))
|
||||
defer server.Close()
|
||||
|
||||
store := openReconcileBackgroundTestStore(t)
|
||||
defer closeAppTestStore(t, store)
|
||||
|
||||
batchID, hostPK, _ := seedReconcileBackgroundRuntimeImport(t, store, server.URL)
|
||||
|
||||
if err := runReconcileBackgroundSweep(context.Background(), store, 10*time.Minute, time.Now()); err != nil {
|
||||
t.Fatalf("runReconcileBackgroundSweep() error = %v", err)
|
||||
}
|
||||
|
||||
providers, err := store.Providers().ListByProviderID(context.Background(), "deepseek")
|
||||
if err != nil {
|
||||
t.Fatalf("Providers().ListByProviderID() error = %v", err)
|
||||
}
|
||||
runs, err := store.ReconcileRuns().GetByProviderIDAndHostID(context.Background(), providers[0].ID, hostPK)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcileRuns().GetByProviderIDAndHostID() error = %v", err)
|
||||
}
|
||||
if len(runs) != 1 {
|
||||
t.Fatalf("reconcile runs = %d, want 1", len(runs))
|
||||
}
|
||||
if runs[0].BatchID != batchID {
|
||||
t.Fatalf("reconcile batch_id = %d, want %d", runs[0].BatchID, batchID)
|
||||
}
|
||||
if runs[0].Status == "" {
|
||||
t.Fatal("reconcile status = empty, want persisted result")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReconcileBackgroundSweepSkipsRecentReconcileRun(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(newBatchImportActionStubServer(t))
|
||||
defer server.Close()
|
||||
|
||||
store := openReconcileBackgroundTestStore(t)
|
||||
defer closeAppTestStore(t, store)
|
||||
|
||||
batchID, hostPK, providerPK := seedReconcileBackgroundRuntimeImport(t, store, server.URL)
|
||||
if _, err := store.ReconcileRuns().Create(context.Background(), sqlite.ReconcileRun{
|
||||
BatchID: batchID,
|
||||
HostID: hostPK,
|
||||
ProviderID: providerPK,
|
||||
Status: "active",
|
||||
SummaryJSON: `{"seed":true}`,
|
||||
}); err != nil {
|
||||
t.Fatalf("ReconcileRuns().Create() error = %v", err)
|
||||
}
|
||||
|
||||
if err := runReconcileBackgroundSweep(context.Background(), store, 10*time.Minute, time.Now()); err != nil {
|
||||
t.Fatalf("runReconcileBackgroundSweep() error = %v", err)
|
||||
}
|
||||
|
||||
runs, err := store.ReconcileRuns().GetByProviderIDAndHostID(context.Background(), providerPK, hostPK)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcileRuns().GetByProviderIDAndHostID() error = %v", err)
|
||||
}
|
||||
if len(runs) != 1 {
|
||||
t.Fatalf("reconcile runs = %d, want 1 recent run only", len(runs))
|
||||
}
|
||||
}
|
||||
|
||||
func openReconcileBackgroundTestStore(t *testing.T) *sqlite.DB {
|
||||
t.Helper()
|
||||
|
||||
dsn := fmt.Sprintf("file:%s?_busy_timeout=5000&_pragma=foreign_keys(0)", filepath.ToSlash(filepath.Join(t.TempDir(), "state.db")))
|
||||
store, err := sqlite.Open(context.Background(), dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("sqlite.Open() error = %v", err)
|
||||
}
|
||||
if _, err := store.SQLDB().Exec("PRAGMA foreign_keys = OFF"); err != nil {
|
||||
t.Fatalf("disable foreign keys pragma error = %v", err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
func seedReconcileBackgroundRuntimeImport(t *testing.T, store *sqlite.DB, baseURL string) (int64, int64, int64) {
|
||||
t.Helper()
|
||||
|
||||
hostPK, err := store.Hosts().Create(context.Background(), sqlite.Host{
|
||||
HostID: "host-1",
|
||||
BaseURL: baseURL,
|
||||
HostVersion: "0.1.126",
|
||||
CapabilityProbeJSON: "{}",
|
||||
AuthType: "apikey",
|
||||
AuthToken: "host-token",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Hosts().Create() error = %v", err)
|
||||
}
|
||||
|
||||
client, err := newSub2APIClient(baseURL, CreateHostAuth{Type: "apikey", Token: "host-token"})
|
||||
if err != nil {
|
||||
t.Fatalf("newSub2APIClient() error = %v", err)
|
||||
}
|
||||
|
||||
loadedPack := pack.LoadedPack{
|
||||
Manifest: pack.Manifest{
|
||||
PackID: "openai-cn-pack",
|
||||
Version: "1.0.0",
|
||||
Vendor: "OpenAI CN",
|
||||
TargetHost: "sub2api",
|
||||
MinHostVersion: "0.1.126",
|
||||
MaxHostVersion: "0.2.x",
|
||||
},
|
||||
Providers: []pack.ProviderManifest{{
|
||||
ProviderID: "deepseek",
|
||||
DisplayName: "DeepSeek",
|
||||
BaseURL: "https://api.deepseek.example",
|
||||
Platform: "openai",
|
||||
AccountType: "openai",
|
||||
DefaultModels: []string{"kimi-k2.6"},
|
||||
SmokeTestModel: "kimi-k2.6",
|
||||
GroupTemplate: pack.GroupTemplate{Name: "DeepSeek 默认分组", RateMultiplier: 1},
|
||||
ChannelTemplate: pack.ChannelTemplate{
|
||||
Name: "DeepSeek 默认渠道",
|
||||
ModelMapping: map[string]string{"kimi-k2.6": "kimi-k2.6"},
|
||||
},
|
||||
PlanTemplate: pack.PlanTemplate{Name: "DeepSeek 默认套餐", Price: 0, ValidityDays: 30, ValidityUnit: "day"},
|
||||
Import: pack.ImportOptions{SupportsMultiKey: true, SupportsStrict: true, SupportsPartial: true},
|
||||
}},
|
||||
Checksum: "checksum-1",
|
||||
}
|
||||
|
||||
result, err := provision.NewRuntimeImportService(store, client).Import(context.Background(), provision.RuntimeImportRequest{
|
||||
HostID: "host-1",
|
||||
HostBaseURL: baseURL,
|
||||
Pack: loadedPack,
|
||||
Provider: loadedPack.Providers[0],
|
||||
Mode: provision.ImportModePartial,
|
||||
Keys: []string{"entry-key"},
|
||||
Access: provision.AccessRequest{
|
||||
Mode: provision.AccessModeSelfService,
|
||||
ProbeAPIKey: "gateway-key",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RuntimeImportService.Import() error = %v", err)
|
||||
}
|
||||
|
||||
packRow, err := store.Packs().GetByPackID(context.Background(), loadedPack.Manifest.PackID)
|
||||
if err != nil {
|
||||
t.Fatalf("Packs().GetByPackID() error = %v", err)
|
||||
}
|
||||
providerRow, err := store.Providers().GetByPackIDAndProviderID(context.Background(), packRow.ID, loadedPack.Providers[0].ProviderID)
|
||||
if err != nil {
|
||||
t.Fatalf("Providers().GetByPackIDAndProviderID() error = %v", err)
|
||||
}
|
||||
|
||||
return result.BatchID, hostPK, providerRow.ID
|
||||
}
|
||||
@@ -20,6 +20,10 @@ type ConfirmationItemStore interface {
|
||||
Upsert(ctx context.Context, item sqlite.ImportRunItem) error
|
||||
}
|
||||
|
||||
type ConfirmationLeaseClaimer interface {
|
||||
TryAcquireLease(ctx context.Context, itemID, workerID string, now time.Time, leaseDuration time.Duration) (sqlite.ImportRunItem, bool, error)
|
||||
}
|
||||
|
||||
type ConfirmationEventStore interface {
|
||||
Append(ctx context.Context, event sqlite.ImportRunItemEvent) error
|
||||
}
|
||||
@@ -53,6 +57,16 @@ func (w ConfirmationWorker) Tick(ctx context.Context, now time.Time) error {
|
||||
if !isConfirmationCandidate(item, now) {
|
||||
continue
|
||||
}
|
||||
if claimer, ok := w.ItemStore.(ConfirmationLeaseClaimer); ok {
|
||||
claimedItem, claimed, err := claimer.TryAcquireLease(ctx, item.ItemID, w.WorkerID, now, w.LeaseDuration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !claimed {
|
||||
continue
|
||||
}
|
||||
item = claimedItem
|
||||
}
|
||||
if err := w.ConfirmItem(ctx, item, now); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -66,9 +80,11 @@ func (w ConfirmationWorker) ConfirmItem(ctx context.Context, item sqlite.ImportR
|
||||
return err
|
||||
}
|
||||
|
||||
item.ConfirmationAttempts++
|
||||
item.LeaseOwner = strings.TrimSpace(w.WorkerID)
|
||||
item.LeaseUntil = now.Add(defaultDuration(w.LeaseDuration, time.Minute)).Format(time.RFC3339)
|
||||
if strings.TrimSpace(item.LeaseOwner) == "" {
|
||||
item.ConfirmationAttempts++
|
||||
item.LeaseOwner = strings.TrimSpace(w.WorkerID)
|
||||
item.LeaseUntil = now.Add(defaultDuration(w.LeaseDuration, time.Minute)).Format(time.RFC3339)
|
||||
}
|
||||
|
||||
switch {
|
||||
case result.StatusCode >= 200 && result.StatusCode < 300:
|
||||
|
||||
@@ -3,6 +3,8 @@ package batch
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -160,6 +162,50 @@ func TestConfirmationWorker(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent workers do not both call confirmer before lease is persisted", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
now := time.Date(2026, 5, 22, 13, 3, 30, 0, time.UTC)
|
||||
store := newFakeConfirmationStore([]sqlite.ImportRunItem{
|
||||
{ItemID: "shared", RunID: "run-1", CurrentStage: "confirm", ConfirmationStatus: "pending"},
|
||||
})
|
||||
|
||||
started := make(chan struct{}, 2)
|
||||
release := make(chan struct{})
|
||||
var calls atomic.Int32
|
||||
|
||||
confirmer := func(ctx context.Context, item sqlite.ImportRunItem) (ConfirmationResult, error) {
|
||||
calls.Add(1)
|
||||
started <- struct{}{}
|
||||
<-release
|
||||
return ConfirmationResult{StatusCode: 200}, nil
|
||||
}
|
||||
|
||||
workerA := ConfirmationWorker{WorkerID: "worker-a", ItemStore: store, EventStore: store, LeaseDuration: time.Minute, RetryDelay: time.Second, Confirmer: confirmer}
|
||||
workerB := ConfirmationWorker{WorkerID: "worker-b", ItemStore: store, EventStore: store, LeaseDuration: time.Minute, RetryDelay: time.Second, Confirmer: confirmer}
|
||||
|
||||
errCh := make(chan error, 2)
|
||||
go func() { errCh <- workerA.Tick(context.Background(), now) }()
|
||||
go func() { errCh <- workerB.Tick(context.Background(), now) }()
|
||||
|
||||
<-started
|
||||
select {
|
||||
case <-started:
|
||||
t.Fatal("second worker reached confirmer before lease was acquired")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(release)
|
||||
for range 2 {
|
||||
if err := <-errCh; err != nil {
|
||||
t.Fatalf("Tick() error = %v", err)
|
||||
}
|
||||
}
|
||||
if got := calls.Load(); got != 1 {
|
||||
t.Fatalf("confirmer calls = %d, want 1", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reactivated account metadata is preserved", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -200,6 +246,7 @@ func TestConfirmationWorker(t *testing.T) {
|
||||
}
|
||||
|
||||
type fakeConfirmationStore struct {
|
||||
mu sync.Mutex
|
||||
items map[string]sqlite.ImportRunItem
|
||||
processed []string
|
||||
events []sqlite.ImportRunItemEvent
|
||||
@@ -217,6 +264,9 @@ func newFakeConfirmationStore(items []sqlite.ImportRunItem) *fakeConfirmationSto
|
||||
}
|
||||
|
||||
func (f *fakeConfirmationStore) List(ctx context.Context) ([]sqlite.ImportRunItem, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
items := make([]sqlite.ImportRunItem, 0, len(f.items))
|
||||
for _, item := range f.items {
|
||||
items = append(items, item)
|
||||
@@ -225,12 +275,36 @@ func (f *fakeConfirmationStore) List(ctx context.Context) ([]sqlite.ImportRunIte
|
||||
}
|
||||
|
||||
func (f *fakeConfirmationStore) Upsert(ctx context.Context, item sqlite.ImportRunItem) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
f.items[item.ItemID] = item
|
||||
f.processed = append(f.processed, item.ItemID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeConfirmationStore) TryAcquireLease(ctx context.Context, itemID, workerID string, now time.Time, leaseDuration time.Duration) (sqlite.ImportRunItem, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
item, ok := f.items[itemID]
|
||||
if !ok {
|
||||
return sqlite.ImportRunItem{}, false, nil
|
||||
}
|
||||
if !isConfirmationCandidate(item, now) {
|
||||
return sqlite.ImportRunItem{}, false, nil
|
||||
}
|
||||
item.ConfirmationAttempts++
|
||||
item.LeaseOwner = workerID
|
||||
item.LeaseUntil = now.Add(leaseDuration).Format(time.RFC3339)
|
||||
f.items[itemID] = item
|
||||
return item, true, nil
|
||||
}
|
||||
|
||||
func (f *fakeConfirmationStore) Append(ctx context.Context, event sqlite.ImportRunItemEvent) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
f.events = append(f.events, event)
|
||||
return nil
|
||||
}
|
||||
@@ -238,6 +312,9 @@ func (f *fakeConfirmationStore) Append(ctx context.Context, event sqlite.ImportR
|
||||
func (f *fakeConfirmationStore) mustItem(t *testing.T, itemID string) sqlite.ImportRunItem {
|
||||
t.Helper()
|
||||
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
item, ok := f.items[itemID]
|
||||
if !ok {
|
||||
t.Fatalf("item %q not found", itemID)
|
||||
|
||||
@@ -19,12 +19,15 @@ type BatchImportEntry struct {
|
||||
}
|
||||
|
||||
type BatchImportRunRequest struct {
|
||||
RunID string
|
||||
Mode string
|
||||
AccessMode string
|
||||
HostID string
|
||||
HostBaseURL string
|
||||
Entries []BatchImportEntry
|
||||
RunID string
|
||||
Mode string
|
||||
AccessMode string
|
||||
HostID string
|
||||
HostBaseURL string
|
||||
SubscriptionUsers []string
|
||||
SubscriptionDays int
|
||||
ProbeAPIKey string
|
||||
Entries []BatchImportEntry
|
||||
}
|
||||
|
||||
type BatchImportRunResult struct {
|
||||
@@ -34,6 +37,7 @@ type BatchImportRunResult struct {
|
||||
|
||||
type RunStateStore interface {
|
||||
Create(ctx context.Context, run sqlite.ImportRun) error
|
||||
Update(ctx context.Context, run sqlite.ImportRun) error
|
||||
}
|
||||
|
||||
type ItemStateStore interface {
|
||||
@@ -114,11 +118,15 @@ func (s BatchImportService) StartRun(ctx context.Context, req BatchImportRunRequ
|
||||
}
|
||||
|
||||
if err := s.RunStore.Create(ctx, sqlite.ImportRun{
|
||||
RunID: runID,
|
||||
Mode: strings.TrimSpace(req.Mode),
|
||||
AccessMode: strings.TrimSpace(req.AccessMode),
|
||||
State: string(RunStateRunning),
|
||||
TotalItems: len(req.Entries),
|
||||
RunID: runID,
|
||||
HostID: strings.TrimSpace(req.HostID),
|
||||
Mode: strings.TrimSpace(req.Mode),
|
||||
AccessMode: strings.TrimSpace(req.AccessMode),
|
||||
SubscriptionUsersJSON: mustMarshalJSON(req.SubscriptionUsers, "[]"),
|
||||
SubscriptionDays: req.SubscriptionDays,
|
||||
ProbeAPIKey: strings.TrimSpace(req.ProbeAPIKey),
|
||||
State: string(RunStateRunning),
|
||||
TotalItems: len(req.Entries),
|
||||
}); err != nil {
|
||||
return BatchImportRunResult{}, err
|
||||
}
|
||||
@@ -152,17 +160,26 @@ func (s BatchImportService) StartRun(ctx context.Context, req BatchImportRunRequ
|
||||
|
||||
modelsResult, err := s.ProbeModels(ctx, entry.BaseURL, entry.APIKey)
|
||||
if err != nil {
|
||||
return BatchImportRunResult{}, err
|
||||
if failErr := s.failRun(ctx, req, initialItem, ItemStageProbe, err); failErr != nil {
|
||||
return BatchImportRunResult{}, failErr
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
rawModels := append([]string(nil), modelsResult.RawModels...)
|
||||
capabilityProfile, err := s.ProbeCapabilities(ctx, entry.BaseURL, entry.APIKey, rawModels)
|
||||
if err != nil {
|
||||
return BatchImportRunResult{}, err
|
||||
if failErr := s.failRun(ctx, req, initialItem, ItemStageProbe, err); failErr != nil {
|
||||
return BatchImportRunResult{}, failErr
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
routingStrategy := BuildImportRoutingStrategy(capabilityProfile)
|
||||
resolvedSmokeModel, recommendedModels, err := probe.ResolveSmokeModel(entry.RequestedModels, rawModels, capabilityProfile)
|
||||
if err != nil {
|
||||
return BatchImportRunResult{}, err
|
||||
if failErr := s.failRun(ctx, req, initialItem, ItemStageProbe, err); failErr != nil {
|
||||
return BatchImportRunResult{}, failErr
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
canonicalFamilies := uniqueCanonicalFamilies(rawModels)
|
||||
@@ -176,7 +193,10 @@ func (s BatchImportService) StartRun(ctx context.Context, req BatchImportRunRequ
|
||||
CanonicalModelFamilies: canonicalFamilies,
|
||||
})
|
||||
if err != nil {
|
||||
return BatchImportRunResult{}, err
|
||||
if failErr := s.failRun(ctx, req, initialItem, ItemStageProbe, err); failErr != nil {
|
||||
return BatchImportRunResult{}, failErr
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,18 +237,27 @@ func (s BatchImportService) StartRun(ctx context.Context, req BatchImportRunRequ
|
||||
patchContract := ModelMappingDelta(reuseLookup.ExistingModelMapping, probe.BuildAliasTable(rawModels))
|
||||
if shouldPatchAliases(reuseLookup.ExistingModelMapping, patchContract.ModelMapping) {
|
||||
if s.Provisioner == nil {
|
||||
return BatchImportRunResult{}, fmt.Errorf("provisioner is required for patch-only flow")
|
||||
if failErr := s.failRun(ctx, req, initialItem, ItemStageProvision, fmt.Errorf("provisioner is required for patch-only flow")); failErr != nil {
|
||||
return BatchImportRunResult{}, failErr
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
if err := s.Provisioner.Patch(ctx, PatchProvisionRequest{
|
||||
ProviderID: reuseDecision.ReusedFromProviderID,
|
||||
Contract: patchContract,
|
||||
}); err != nil {
|
||||
return BatchImportRunResult{}, err
|
||||
if failErr := s.failRun(ctx, req, initialItem, ItemStageProvision, err); failErr != nil {
|
||||
return BatchImportRunResult{}, failErr
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if s.Provisioner == nil {
|
||||
return BatchImportRunResult{}, fmt.Errorf("provisioner is required")
|
||||
if failErr := s.failRun(ctx, req, initialItem, ItemStageProvision, fmt.Errorf("provisioner is required")); failErr != nil {
|
||||
return BatchImportRunResult{}, failErr
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
provisionResult, err := s.Provisioner.Provision(ctx, ProvisionRequest{
|
||||
RunID: runID,
|
||||
@@ -240,7 +269,10 @@ func (s BatchImportService) StartRun(ctx context.Context, req BatchImportRunRequ
|
||||
CapabilityProfile: capabilityProfile,
|
||||
})
|
||||
if err != nil {
|
||||
return BatchImportRunResult{}, err
|
||||
if failErr := s.failRun(ctx, req, initialItem, ItemStageProvision, err); failErr != nil {
|
||||
return BatchImportRunResult{}, failErr
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
finalItem.LegacyBatchID = provisionResult.LegacyBatchID
|
||||
finalItem.LegacyProviderID = strings.TrimSpace(provisionResult.LegacyProviderID)
|
||||
@@ -254,6 +286,35 @@ func (s BatchImportService) StartRun(ctx context.Context, req BatchImportRunRequ
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s BatchImportService) failRun(ctx context.Context, req BatchImportRunRequest, item sqlite.ImportRunItem, stage ItemStage, cause error) error {
|
||||
item.CurrentStage = string(ItemStageDone)
|
||||
item.ConfirmationStatus = string(ConfirmationFailed)
|
||||
item.AccessStatus = string(AccessStatusBroken)
|
||||
item.LastErrorStage = string(stage)
|
||||
item.LastError = strings.TrimSpace(cause.Error())
|
||||
item.LeaseOwner = ""
|
||||
item.LeaseUntil = ""
|
||||
item.NextRetryAt = ""
|
||||
if err := s.ItemStore.Upsert(ctx, item); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.RunStore.Update(ctx, sqlite.ImportRun{
|
||||
RunID: strings.TrimSpace(req.RunID),
|
||||
HostID: strings.TrimSpace(req.HostID),
|
||||
Mode: strings.TrimSpace(req.Mode),
|
||||
AccessMode: strings.TrimSpace(req.AccessMode),
|
||||
SubscriptionUsersJSON: mustMarshalJSON(req.SubscriptionUsers, "[]"),
|
||||
SubscriptionDays: req.SubscriptionDays,
|
||||
ProbeAPIKey: strings.TrimSpace(req.ProbeAPIKey),
|
||||
State: string(RunStateFailed),
|
||||
TotalItems: len(req.Entries),
|
||||
CompletedItems: 1,
|
||||
BrokenItems: 1,
|
||||
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func uniqueCanonicalFamilies(rawModels []string) []string {
|
||||
seen := make(map[string]struct{}, len(rawModels))
|
||||
families := make([]string, 0, len(rawModels))
|
||||
|
||||
@@ -115,6 +115,7 @@ func TestBatchImport_StartRun(t *testing.T) {
|
||||
RunID: "run-2",
|
||||
Mode: "strict",
|
||||
AccessMode: "subscription",
|
||||
HostID: "host-1",
|
||||
Entries: []BatchImportEntry{
|
||||
{BaseURL: "https://api.kimi.com/v1", APIKey: "sk-live", RequestedModels: []string{"kimi 2.6"}},
|
||||
},
|
||||
@@ -170,6 +171,7 @@ func TestBatchImport_StartRun(t *testing.T) {
|
||||
RunID: "run-3",
|
||||
Mode: "strict",
|
||||
AccessMode: "subscription",
|
||||
HostID: "host-1",
|
||||
Entries: []BatchImportEntry{
|
||||
{BaseURL: "https://api.kimi.com/v1", APIKey: "sk-live", RequestedModels: []string{"kimi 2.6"}},
|
||||
},
|
||||
@@ -221,6 +223,7 @@ func TestBatchImport_StartRun(t *testing.T) {
|
||||
RunID: "run-4",
|
||||
Mode: "strict",
|
||||
AccessMode: "subscription",
|
||||
HostID: "host-1",
|
||||
Entries: []BatchImportEntry{
|
||||
{BaseURL: "https://api.kimi.com/v1", APIKey: "sk-live", RequestedModels: []string{"kimi 2.6"}},
|
||||
},
|
||||
@@ -243,10 +246,71 @@ func TestBatchImport_StartRun(t *testing.T) {
|
||||
t.Fatal("ProvisionReused = false, want true for patch-only flow")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("probe failure marks run failed instead of leaving running half state", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
runStore := &fakeRunStore{}
|
||||
itemStore := &fakeItemStore{}
|
||||
service := BatchImportService{
|
||||
RunStore: runStore,
|
||||
ItemStore: itemStore,
|
||||
ProbeModels: func(context.Context, string, string) (*probe.ModelsResult, error) {
|
||||
return nil, context.DeadlineExceeded
|
||||
},
|
||||
ProbeCapabilities: func(context.Context, string, string, []string) (*probe.CapabilityProfile, error) {
|
||||
t.Fatal("ProbeCapabilities should not be called after probe failure")
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
|
||||
result, err := service.StartRun(context.Background(), BatchImportRunRequest{
|
||||
RunID: "run-probe-fail",
|
||||
Mode: "strict",
|
||||
AccessMode: "self_service",
|
||||
HostID: "host-1",
|
||||
Entries: []BatchImportEntry{
|
||||
{BaseURL: "https://api.deepseek.com/v1", APIKey: "sk-live", RequestedModels: []string{"DeepSeek V4 Pro"}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StartRun() error = %v, want persisted failed run without transport error", err)
|
||||
}
|
||||
if result.RunID != "run-probe-fail" {
|
||||
t.Fatalf("result.RunID = %q, want run-probe-fail", result.RunID)
|
||||
}
|
||||
if len(runStore.updated) == 0 {
|
||||
t.Fatal("run store was not updated to failed state")
|
||||
}
|
||||
gotRun := runStore.updated[len(runStore.updated)-1]
|
||||
if gotRun.State != string(RunStateFailed) {
|
||||
t.Fatalf("run.State = %q, want failed", gotRun.State)
|
||||
}
|
||||
if gotRun.CompletedItems != 1 || gotRun.BrokenItems != 1 {
|
||||
t.Fatalf("run counters = %+v, want completed_items=1 broken_items=1", gotRun)
|
||||
}
|
||||
if len(itemStore.upserts) < 2 {
|
||||
t.Fatalf("item upserts = %d, want initial + failed terminal state", len(itemStore.upserts))
|
||||
}
|
||||
gotItem := itemStore.upserts[len(itemStore.upserts)-1]
|
||||
if gotItem.CurrentStage != string(ItemStageDone) {
|
||||
t.Fatalf("item.CurrentStage = %q, want done", gotItem.CurrentStage)
|
||||
}
|
||||
if gotItem.ConfirmationStatus != string(ConfirmationFailed) {
|
||||
t.Fatalf("item.ConfirmationStatus = %q, want failed", gotItem.ConfirmationStatus)
|
||||
}
|
||||
if gotItem.AccessStatus != string(AccessStatusBroken) {
|
||||
t.Fatalf("item.AccessStatus = %q, want broken", gotItem.AccessStatus)
|
||||
}
|
||||
if gotItem.LastErrorStage != string(ItemStageProbe) {
|
||||
t.Fatalf("item.LastErrorStage = %q, want probe", gotItem.LastErrorStage)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type fakeRunStore struct {
|
||||
created []sqlite.ImportRun
|
||||
updated []sqlite.ImportRun
|
||||
}
|
||||
|
||||
func (f *fakeRunStore) Create(ctx context.Context, run sqlite.ImportRun) error {
|
||||
@@ -254,6 +318,11 @@ func (f *fakeRunStore) Create(ctx context.Context, run sqlite.ImportRun) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRunStore) Update(ctx context.Context, run sqlite.ImportRun) error {
|
||||
f.updated = append(f.updated, run)
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeItemStore struct {
|
||||
upserts []sqlite.ImportRunItem
|
||||
}
|
||||
|
||||
@@ -4,15 +4,19 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
EnvListenAddr = "SUB2API_CRM_LISTEN_ADDR"
|
||||
EnvSQLiteDSN = "SUB2API_CRM_SQLITE_DSN"
|
||||
EnvAdminToken = "SUB2API_CRM_ADMIN_TOKEN"
|
||||
EnvListenAddr = "SUB2API_CRM_LISTEN_ADDR"
|
||||
EnvSQLiteDSN = "SUB2API_CRM_SQLITE_DSN"
|
||||
EnvAdminToken = "SUB2API_CRM_ADMIN_TOKEN"
|
||||
EnvReconcileWorkerEnabled = "SUB2API_CRM_RECONCILE_WORKER_ENABLED"
|
||||
EnvReconcilePollInterval = "SUB2API_CRM_RECONCILE_POLL_INTERVAL"
|
||||
|
||||
DefaultListenAddr = ":8080"
|
||||
DefaultSQLiteDSN = "file:sub2api-cn-relay-manager.db?_foreign_keys=on&_busy_timeout=5000"
|
||||
DefaultListenAddr = ":8080"
|
||||
DefaultSQLiteDSN = "file:sub2api-cn-relay-manager.db?_foreign_keys=on&_busy_timeout=5000"
|
||||
DefaultReconcilePollInterval = 10 * time.Minute
|
||||
)
|
||||
|
||||
type ServerConfig struct {
|
||||
@@ -23,9 +27,15 @@ type DatabaseConfig struct {
|
||||
SQLiteDSN string
|
||||
}
|
||||
|
||||
type ReconcileConfig struct {
|
||||
WorkerEnabled bool
|
||||
PollInterval time.Duration
|
||||
}
|
||||
|
||||
type StartupConfig struct {
|
||||
Server ServerConfig
|
||||
Database DatabaseConfig
|
||||
Server ServerConfig
|
||||
Database DatabaseConfig
|
||||
Reconcile ReconcileConfig
|
||||
}
|
||||
|
||||
func LoadStartupFromEnv() (StartupConfig, error) {
|
||||
@@ -33,6 +43,10 @@ func LoadStartupFromEnv() (StartupConfig, error) {
|
||||
}
|
||||
|
||||
func loadStartupFromLookupEnv(lookup func(string) (string, bool)) (StartupConfig, error) {
|
||||
reconcilePollInterval, err := readOptionalDurationEnv(lookup, EnvReconcilePollInterval, DefaultReconcilePollInterval)
|
||||
if err != nil {
|
||||
return StartupConfig{}, err
|
||||
}
|
||||
cfg := StartupConfig{
|
||||
Server: ServerConfig{
|
||||
ListenAddr: readOptionalEnv(lookup, EnvListenAddr, DefaultListenAddr),
|
||||
@@ -40,6 +54,10 @@ func loadStartupFromLookupEnv(lookup func(string) (string, bool)) (StartupConfig
|
||||
Database: DatabaseConfig{
|
||||
SQLiteDSN: readOptionalEnv(lookup, EnvSQLiteDSN, DefaultSQLiteDSN),
|
||||
},
|
||||
Reconcile: ReconcileConfig{
|
||||
WorkerEnabled: readOptionalBoolEnv(lookup, EnvReconcileWorkerEnabled, false),
|
||||
PollInterval: reconcilePollInterval,
|
||||
},
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
@@ -80,3 +98,33 @@ func readRequiredEnv(lookup func(string) (string, bool), key string) string {
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
func readOptionalBoolEnv(lookup func(string) (string, bool), key string, defaultValue bool) bool {
|
||||
value, ok := lookup(key)
|
||||
if !ok {
|
||||
return defaultValue
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "1", "true", "yes", "on":
|
||||
return true
|
||||
case "0", "false", "no", "off", "":
|
||||
return false
|
||||
default:
|
||||
return defaultValue
|
||||
}
|
||||
}
|
||||
|
||||
func readOptionalDurationEnv(lookup func(string) (string, bool), key string, defaultValue time.Duration) (time.Duration, error) {
|
||||
value, ok := lookup(key)
|
||||
if !ok || strings.TrimSpace(value) == "" {
|
||||
return defaultValue, nil
|
||||
}
|
||||
duration, err := time.ParseDuration(strings.TrimSpace(value))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%s: parse duration: %w", key, err)
|
||||
}
|
||||
if duration <= 0 {
|
||||
return 0, fmt.Errorf("%s: duration must be positive", key)
|
||||
}
|
||||
return duration, nil
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestReadOptionalEnv(t *testing.T) {
|
||||
@@ -62,6 +63,10 @@ func TestLoadStartupFromLookupEnv(t *testing.T) {
|
||||
return ":9090", true
|
||||
case EnvSQLiteDSN:
|
||||
return "/data/db.sqlite", true
|
||||
case EnvReconcileWorkerEnabled:
|
||||
return "true", true
|
||||
case EnvReconcilePollInterval:
|
||||
return "15m", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
@@ -76,6 +81,12 @@ func TestLoadStartupFromLookupEnv(t *testing.T) {
|
||||
if cfg.Database.SQLiteDSN != "/data/db.sqlite" {
|
||||
t.Fatalf("SQLiteDSN = %q, want %q", cfg.Database.SQLiteDSN, "/data/db.sqlite")
|
||||
}
|
||||
if !cfg.Reconcile.WorkerEnabled {
|
||||
t.Fatal("WorkerEnabled = false, want true")
|
||||
}
|
||||
if cfg.Reconcile.PollInterval != 15*time.Minute {
|
||||
t.Fatalf("PollInterval = %s, want 15m", cfg.Reconcile.PollInterval)
|
||||
}
|
||||
})
|
||||
t.Run("default values", func(t *testing.T) {
|
||||
lookup := func(k string) (string, bool) {
|
||||
@@ -91,6 +102,23 @@ func TestLoadStartupFromLookupEnv(t *testing.T) {
|
||||
if cfg.Database.SQLiteDSN != DefaultSQLiteDSN {
|
||||
t.Fatalf("SQLiteDSN = %q, want %q", cfg.Database.SQLiteDSN, DefaultSQLiteDSN)
|
||||
}
|
||||
if cfg.Reconcile.WorkerEnabled {
|
||||
t.Fatal("WorkerEnabled = true, want false by default")
|
||||
}
|
||||
if cfg.Reconcile.PollInterval != DefaultReconcilePollInterval {
|
||||
t.Fatalf("PollInterval = %s, want %s", cfg.Reconcile.PollInterval, DefaultReconcilePollInterval)
|
||||
}
|
||||
})
|
||||
t.Run("invalid reconcile interval", func(t *testing.T) {
|
||||
lookup := func(k string) (string, bool) {
|
||||
if k == EnvReconcilePollInterval {
|
||||
return "not-a-duration", true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
if _, err := loadStartupFromLookupEnv(lookup); err == nil {
|
||||
t.Fatal("loadStartupFromLookupEnv() error = nil, want invalid interval")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
36
internal/host/sub2api/account_capability_repair.go
Normal file
36
internal/host/sub2api/account_capability_repair.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package sub2api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (c *Client) DisableOpenAIResponsesAPI(ctx context.Context, accountIDs []string) error {
|
||||
seen := map[string]struct{}{}
|
||||
for _, rawID := range accountIDs {
|
||||
accountID := strings.TrimSpace(rawID)
|
||||
if accountID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[accountID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[accountID] = struct{}{}
|
||||
|
||||
path := "/api/v1/admin/accounts/" + accountID
|
||||
payload := map[string]any{
|
||||
"extra": map[string]any{
|
||||
"openai_responses_supported": false,
|
||||
},
|
||||
}
|
||||
statusCode, _, body, err := c.perform(ctx, http.MethodPut, path, payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if statusCode < http.StatusOK || statusCode >= http.StatusMultipleChoices {
|
||||
return newHTTPError(http.MethodPut, path, statusCode, body)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -31,6 +31,7 @@ type HostAdapter interface {
|
||||
AssignSubscription(ctx context.Context, req AssignSubscriptionRequest) (SubscriptionRef, error)
|
||||
CheckGatewayAccess(ctx context.Context, req GatewayAccessCheckRequest) (GatewayAccessResult, error)
|
||||
CheckGatewayCompletion(ctx context.Context, req GatewayCompletionCheckRequest) (GatewayCompletionResult, error)
|
||||
DisableOpenAIResponsesAPI(ctx context.Context, accountIDs []string) error
|
||||
ListManagedResources(ctx context.Context, req ListManagedResourcesRequest) (ManagedResourceSnapshot, error)
|
||||
}
|
||||
|
||||
|
||||
@@ -979,6 +979,41 @@ func TestCheckGatewayCompletionWithMock(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisableOpenAIResponsesAPIWithMock(t *testing.T) {
|
||||
var calls []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls = append(calls, r.Method+" "+r.URL.Path)
|
||||
if r.Method != http.MethodPut {
|
||||
t.Fatalf("method = %q, want PUT", r.Method)
|
||||
}
|
||||
var payload struct {
|
||||
Extra map[string]any `json:"extra"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
if got, ok := payload.Extra["openai_responses_supported"].(bool); !ok || got {
|
||||
t.Fatalf("openai_responses_supported = %+v, want false", payload.Extra["openai_responses_supported"])
|
||||
}
|
||||
w.Write([]byte(`{"data":{"id":1}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, _ := NewClient(srv.URL, WithAPIKey("k"))
|
||||
if err := client.DisableOpenAIResponsesAPI(context.Background(), []string{"101", "101", " ", "102"}); err != nil {
|
||||
t.Fatalf("DisableOpenAIResponsesAPI() error = %v", err)
|
||||
}
|
||||
if len(calls) != 2 {
|
||||
t.Fatalf("calls = %v, want 2 unique account updates", calls)
|
||||
}
|
||||
if calls[0] != "PUT /api/v1/admin/accounts/101" {
|
||||
t.Fatalf("first call = %q, want PUT /api/v1/admin/accounts/101", calls[0])
|
||||
}
|
||||
if calls[1] != "PUT /api/v1/admin/accounts/102" {
|
||||
t.Fatalf("second call = %q, want PUT /api/v1/admin/accounts/102", calls[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchCreateAccountsWithMock(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
|
||||
57
internal/provision/batch_detail_service.go
Normal file
57
internal/provision/batch_detail_service.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package provision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
)
|
||||
|
||||
type BatchDetailResult struct {
|
||||
Batch sqlite.ImportBatch
|
||||
Items []sqlite.ImportBatchItem
|
||||
ManagedResources []sqlite.ManagedResource
|
||||
AccessClosures []sqlite.AccessClosureRecord
|
||||
ReconcileRuns []sqlite.ReconcileRun
|
||||
}
|
||||
|
||||
type BatchDetailService struct {
|
||||
store *sqlite.DB
|
||||
}
|
||||
|
||||
func NewBatchDetailService(store *sqlite.DB) *BatchDetailService {
|
||||
return &BatchDetailService{store: store}
|
||||
}
|
||||
|
||||
func (s *BatchDetailService) Get(ctx context.Context, batchID int64) (BatchDetailResult, error) {
|
||||
if s == nil || s.store == nil {
|
||||
return BatchDetailResult{}, fmt.Errorf("store is required")
|
||||
}
|
||||
batch, err := s.store.ImportBatches().GetByID(ctx, batchID)
|
||||
if err != nil {
|
||||
return BatchDetailResult{}, err
|
||||
}
|
||||
items, err := s.store.ImportBatchItems().GetByBatchID(ctx, batchID)
|
||||
if err != nil {
|
||||
return BatchDetailResult{}, err
|
||||
}
|
||||
managedResources, err := s.store.ManagedResources().GetByBatchID(ctx, batchID)
|
||||
if err != nil {
|
||||
return BatchDetailResult{}, err
|
||||
}
|
||||
accessClosures, err := s.store.AccessClosures().GetByBatchID(ctx, batchID)
|
||||
if err != nil {
|
||||
return BatchDetailResult{}, err
|
||||
}
|
||||
reconcileRuns, err := s.store.ReconcileRuns().GetByBatchID(ctx, batchID)
|
||||
if err != nil {
|
||||
return BatchDetailResult{}, err
|
||||
}
|
||||
return BatchDetailResult{
|
||||
Batch: batch,
|
||||
Items: items,
|
||||
ManagedResources: managedResources,
|
||||
AccessClosures: accessClosures,
|
||||
ReconcileRuns: reconcileRuns,
|
||||
}, nil
|
||||
}
|
||||
@@ -142,59 +142,6 @@ func TestAccountIDFromProbeSummary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileServiceRerunAccessClosureWithoutProbeKeyUsesLatestStatus(t *testing.T) {
|
||||
store := openProvisionTestStore(t)
|
||||
defer closeProvisionTestStore(t, store)
|
||||
|
||||
status, checked, err := NewReconcileService(store, &fakeHostAdapter{}).rerunAccessClosure(context.Background(), 1, []sqlite.AccessClosureRecord{{ClosureType: AccessModeSubscription, Status: AccessStatusSubscriptionReady}}, "", "deepseek-chat")
|
||||
if err != nil {
|
||||
t.Fatalf("rerunAccessClosure() error = %v", err)
|
||||
}
|
||||
if checked {
|
||||
t.Fatal("checked = true, want false without probe key")
|
||||
}
|
||||
if status != AccessStatusSubscriptionReady {
|
||||
t.Fatalf("status = %q, want %q", status, AccessStatusSubscriptionReady)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileServiceRerunAccessClosureMarksBrokenWhenGatewayCheckFails(t *testing.T) {
|
||||
store := openProvisionTestStore(t)
|
||||
defer closeProvisionTestStore(t, store)
|
||||
|
||||
hostSeed := &fakeHostAdapter{
|
||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "deepseek-01"}, {ID: "account_2", Name: "deepseek-02"}},
|
||||
testResults: map[string]sub2api.ProbeResult{
|
||||
"account_1": {OK: true, Status: "passed"},
|
||||
"account_2": {OK: true, Status: "passed"},
|
||||
},
|
||||
models: map[string][]sub2api.AccountModel{
|
||||
"account_1": {{ID: "deepseek-chat"}},
|
||||
"account_2": {{ID: "deepseek-chat"}},
|
||||
},
|
||||
gatewayResult: sub2api.GatewayAccessResult{OK: true, StatusCode: 200, HasExpectedModel: true, Models: []string{"deepseek-chat"}},
|
||||
}
|
||||
batchID := seedRuntimeImportForReconcile(t, store, hostSeed)
|
||||
|
||||
host := &fakeHostAdapter{gatewayResult: sub2api.GatewayAccessResult{OK: false, StatusCode: 403, HasExpectedModel: false}}
|
||||
status, checked, err := NewReconcileService(store, host).rerunAccessClosure(context.Background(), batchID, []sqlite.AccessClosureRecord{{ClosureType: AccessModeSelfService, Status: AccessStatusSelfServiceReady}}, "user-key", "deepseek-chat")
|
||||
if err != nil {
|
||||
t.Fatalf("rerunAccessClosure() error = %v", err)
|
||||
}
|
||||
if !checked {
|
||||
t.Fatal("checked = false, want true")
|
||||
}
|
||||
if status != AccessStatusBroken {
|
||||
t.Fatalf("status = %q, want %q", status, AccessStatusBroken)
|
||||
}
|
||||
if got := queryCount(t, store.SQLDB(), "access_closure_records"); got != 2 {
|
||||
t.Fatalf("access_closure_records row count = %d, want 2 after rerun", got)
|
||||
}
|
||||
if host.gatewayProbe.ExpectedModel != "deepseek-chat" {
|
||||
t.Fatalf("ExpectedModel = %q, want deepseek-chat", host.gatewayProbe.ExpectedModel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffManagedResourcesCountsMissingAndExtra(t *testing.T) {
|
||||
missing, extra := diffManagedResources(
|
||||
[]sqlite.ManagedResource{
|
||||
|
||||
@@ -258,11 +258,13 @@ func (s *ImportService) Import(ctx context.Context, req ImportRequest) (report I
|
||||
|
||||
closureService := access.NewService(s.host)
|
||||
gateway, err := closureService.Close(ctx, access.ClosureRequest{
|
||||
Mode: req.Access.Mode,
|
||||
ProbeAPIKey: req.Access.ProbeAPIKey,
|
||||
Subscriptions: toAccessSubscriptionTargets(req.Access.Subscriptions),
|
||||
GroupID: resources.Group.ID,
|
||||
ExpectedModel: req.Provider.SmokeTestModel,
|
||||
Mode: req.Access.Mode,
|
||||
ProbeAPIKey: req.Access.ProbeAPIKey,
|
||||
Subscriptions: toAccessSubscriptionTargets(req.Access.Subscriptions),
|
||||
GroupID: resources.Group.ID,
|
||||
AccountIDs: importedAccountIDs(report.Accounts),
|
||||
ExpectedModel: req.Provider.SmokeTestModel,
|
||||
ResponsesCapabilitySuspect: importedAccountsSuspectResponsesCapabilityMismatch(report.Accounts),
|
||||
})
|
||||
if err != nil {
|
||||
return failOrDegrade(report, req.Mode, err)
|
||||
@@ -326,6 +328,28 @@ func (s *ImportService) ensureManagedResources(ctx context.Context, provider pac
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func importedAccountIDs(accounts []AccountImportResult) []string {
|
||||
ids := make([]string, 0, len(accounts))
|
||||
for _, account := range accounts {
|
||||
if trimmed := strings.TrimSpace(account.Ref.ID); trimmed != "" {
|
||||
ids = append(ids, trimmed)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func importedAccountsSuspectResponsesCapabilityMismatch(accounts []AccountImportResult) bool {
|
||||
for _, account := range accounts {
|
||||
if !account.SmokeModelSeen {
|
||||
continue
|
||||
}
|
||||
if access.SuspectsOpenAIResponsesCapabilityMismatch(account.Probe) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ensureGroup(ctx context.Context, host hostAdapter, existing []sub2api.NamedResource, provider pack.ProviderManifest, accessMode, groupName string) (sub2api.GroupRef, bool, error) {
|
||||
switch len(existing) {
|
||||
case 0:
|
||||
|
||||
@@ -342,6 +342,51 @@ func TestImportServiceRetriesTransientGatewayCompletionFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportServiceRepairsOpenAIResponsesCapabilityMismatchAfterInstall(t *testing.T) {
|
||||
host := &fakeHostAdapter{
|
||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "kimi-a7m-01"}},
|
||||
testResults: map[string]sub2api.ProbeResult{
|
||||
"account_1": {
|
||||
OK: false,
|
||||
Status: "failed",
|
||||
Message: "API returned 403: Forbidden",
|
||||
},
|
||||
},
|
||||
models: map[string][]sub2api.AccountModel{
|
||||
"account_1": {{ID: "deepseek-chat"}},
|
||||
},
|
||||
gatewayResult: sub2api.GatewayAccessResult{
|
||||
OK: true,
|
||||
StatusCode: 200,
|
||||
HasExpectedModel: true,
|
||||
Models: []string{"deepseek-chat"},
|
||||
},
|
||||
completionResults: []sub2api.GatewayCompletionResult{
|
||||
{OK: false, StatusCode: 502, ContentType: "application/json", BodyPreview: `{"error":{"message":"Upstream service temporarily unavailable","type":"upstream_error"}}`},
|
||||
},
|
||||
completionAfterRepair: &sub2api.GatewayCompletionResult{OK: true, StatusCode: 200, ContentType: "application/json"},
|
||||
}
|
||||
|
||||
report, err := NewImportService(host).Import(context.Background(), ImportRequest{
|
||||
Provider: sampleProviderManifest(),
|
||||
Mode: ImportModePartial,
|
||||
Access: AccessRequest{Mode: AccessModeSelfService, ProbeAPIKey: "user-key"},
|
||||
Keys: []string{"key-1"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Import() error = %v", err)
|
||||
}
|
||||
if !report.Gateway.CompletionOK || report.Gateway.CompletionStatus != 200 {
|
||||
t.Fatalf("Gateway completion = %+v, want repaired success", report.Gateway)
|
||||
}
|
||||
if host.disableResponsesCalls != 1 {
|
||||
t.Fatalf("disable responses calls = %d, want 1", host.disableResponsesCalls)
|
||||
}
|
||||
if len(host.disabledResponsesAccountIDs) != 1 || host.disabledResponsesAccountIDs[0] != "account_1" {
|
||||
t.Fatalf("disabled responses account ids = %v, want [account_1]", host.disabledResponsesAccountIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportServiceStrictModeRollsBackCreatedResources(t *testing.T) {
|
||||
host := &fakeHostAdapter{
|
||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1"}, {ID: "account_2"}},
|
||||
@@ -594,35 +639,38 @@ func TestImportKeepsExistingAccountsWhenReplacementValidationFails(t *testing.T)
|
||||
}
|
||||
|
||||
type fakeHostAdapter struct {
|
||||
batchAccounts []sub2api.AccountRef
|
||||
batchCreateReq sub2api.BatchCreateAccountsRequest
|
||||
testResults map[string]sub2api.ProbeResult
|
||||
models map[string][]sub2api.AccountModel
|
||||
gatewayResult sub2api.GatewayAccessResult
|
||||
batchCreateErr error
|
||||
assignErr error
|
||||
gatewayErr error
|
||||
hostVersion string
|
||||
assignedSubscriptions []sub2api.AssignSubscriptionRequest
|
||||
gatewayProbe sub2api.GatewayAccessCheckRequest
|
||||
completionProbe sub2api.GatewayCompletionCheckRequest
|
||||
deletedResources []string
|
||||
managedSnapshot sub2api.ManagedResourceSnapshot
|
||||
listManagedReq sub2api.ListManagedResourcesRequest
|
||||
createGroupCalls int
|
||||
createChannelCalls int
|
||||
updateChannelCalls int
|
||||
createPlanCalls int
|
||||
createGroupReq sub2api.CreateGroupRequest
|
||||
createChannelReq sub2api.CreateChannelRequest
|
||||
updateChannelID string
|
||||
updateChannelReq sub2api.CreateChannelRequest
|
||||
callSequence []string
|
||||
completionCalls int
|
||||
completionResults []sub2api.GatewayCompletionResult
|
||||
completionResult sub2api.GatewayCompletionResult
|
||||
completionErr error
|
||||
testedModels map[string]string
|
||||
batchAccounts []sub2api.AccountRef
|
||||
batchCreateReq sub2api.BatchCreateAccountsRequest
|
||||
testResults map[string]sub2api.ProbeResult
|
||||
models map[string][]sub2api.AccountModel
|
||||
gatewayResult sub2api.GatewayAccessResult
|
||||
batchCreateErr error
|
||||
assignErr error
|
||||
gatewayErr error
|
||||
hostVersion string
|
||||
assignedSubscriptions []sub2api.AssignSubscriptionRequest
|
||||
gatewayProbe sub2api.GatewayAccessCheckRequest
|
||||
completionProbe sub2api.GatewayCompletionCheckRequest
|
||||
deletedResources []string
|
||||
managedSnapshot sub2api.ManagedResourceSnapshot
|
||||
listManagedReq sub2api.ListManagedResourcesRequest
|
||||
createGroupCalls int
|
||||
createChannelCalls int
|
||||
updateChannelCalls int
|
||||
createPlanCalls int
|
||||
createGroupReq sub2api.CreateGroupRequest
|
||||
createChannelReq sub2api.CreateChannelRequest
|
||||
updateChannelID string
|
||||
updateChannelReq sub2api.CreateChannelRequest
|
||||
callSequence []string
|
||||
completionCalls int
|
||||
completionResults []sub2api.GatewayCompletionResult
|
||||
completionResult sub2api.GatewayCompletionResult
|
||||
completionAfterRepair *sub2api.GatewayCompletionResult
|
||||
completionErr error
|
||||
testedModels map[string]string
|
||||
disableResponsesCalls int
|
||||
disabledResponsesAccountIDs []string
|
||||
}
|
||||
|
||||
func (f *fakeHostAdapter) GetHostVersion(context.Context) (string, error) {
|
||||
@@ -724,6 +772,9 @@ func (f *fakeHostAdapter) CheckGatewayCompletion(_ context.Context, req sub2api.
|
||||
if f.completionErr != nil {
|
||||
return sub2api.GatewayCompletionResult{}, f.completionErr
|
||||
}
|
||||
if f.disableResponsesCalls > 0 && f.completionAfterRepair != nil {
|
||||
return *f.completionAfterRepair, nil
|
||||
}
|
||||
if len(f.completionResults) > 0 {
|
||||
idx := f.completionCalls - 1
|
||||
if idx >= len(f.completionResults) {
|
||||
@@ -736,6 +787,11 @@ func (f *fakeHostAdapter) CheckGatewayCompletion(_ context.Context, req sub2api.
|
||||
}
|
||||
return f.completionResult, nil
|
||||
}
|
||||
func (f *fakeHostAdapter) DisableOpenAIResponsesAPI(_ context.Context, accountIDs []string) error {
|
||||
f.disableResponsesCalls++
|
||||
f.disabledResponsesAccountIDs = append([]string(nil), accountIDs...)
|
||||
return nil
|
||||
}
|
||||
func (f *fakeHostAdapter) ListManagedResources(_ context.Context, req sub2api.ListManagedResourcesRequest) (sub2api.ManagedResourceSnapshot, error) {
|
||||
f.listManagedReq = req
|
||||
return sub2api.ManagedResourceSnapshot{
|
||||
|
||||
26
internal/provision/reconcile_compat_test.go
Normal file
26
internal/provision/reconcile_compat_test.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package provision
|
||||
|
||||
import (
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
"sub2api-cn-relay-manager/internal/reconcile"
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
)
|
||||
|
||||
type ReconcileRequest = reconcile.Request
|
||||
type ReconcileResult = reconcile.Result
|
||||
|
||||
func NewReconcileService(store *sqlite.DB, host sub2api.HostAdapter) *reconcile.Service {
|
||||
return reconcile.NewService(store, host)
|
||||
}
|
||||
|
||||
func accountIDFromProbeSummary(summaryJSON string) (string, error) {
|
||||
return reconcile.AccountIDFromProbeSummary(summaryJSON)
|
||||
}
|
||||
|
||||
func diffManagedResources(stored []sqlite.ManagedResource, snapshot sub2api.ManagedResourceSnapshot) (int, int) {
|
||||
return reconcile.DiffManagedResources(stored, snapshot)
|
||||
}
|
||||
|
||||
func deriveHealthyAccessStatus(closureType string) string {
|
||||
return reconcile.DeriveHealthyAccessStatus(closureType)
|
||||
}
|
||||
@@ -162,6 +162,55 @@ func TestReconcileServiceIgnoresAdvisoryProbeFailureWhenModelsAndGatewayAreHealt
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileServiceRepairsOpenAIResponsesCapabilityMismatch(t *testing.T) {
|
||||
store := openProvisionTestStore(t)
|
||||
defer closeProvisionTestStore(t, store)
|
||||
|
||||
host := &fakeHostAdapter{
|
||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "deepseek-01"}, {ID: "account_2", Name: "deepseek-02"}},
|
||||
testResults: map[string]sub2api.ProbeResult{
|
||||
"account_1": {OK: false, Status: "failed", Message: "API returned 403: Forbidden"},
|
||||
"account_2": {OK: false, Status: "failed", Message: "API returned 403: Forbidden"},
|
||||
},
|
||||
models: map[string][]sub2api.AccountModel{
|
||||
"account_1": {{ID: "deepseek-chat"}},
|
||||
"account_2": {{ID: "deepseek-chat"}},
|
||||
},
|
||||
gatewayResult: sub2api.GatewayAccessResult{OK: true, StatusCode: 200, HasExpectedModel: true, Models: []string{"deepseek-chat"}},
|
||||
completionResults: []sub2api.GatewayCompletionResult{
|
||||
{OK: false, StatusCode: 502, ContentType: "application/json", BodyPreview: `{"error":{"message":"Upstream service temporarily unavailable","type":"upstream_error"}}`},
|
||||
},
|
||||
completionAfterRepair: &sub2api.GatewayCompletionResult{OK: true, StatusCode: 200, ContentType: "application/json"},
|
||||
}
|
||||
|
||||
seedRuntimeImportForReconcile(t, store, host)
|
||||
host.managedSnapshot = sub2api.ManagedResourceSnapshot{
|
||||
Groups: []sub2api.NamedResource{{ID: "group_1", Name: "DeepSeek 默认分组-self-service"}},
|
||||
Channels: []sub2api.NamedResource{{ID: "channel_1", Name: "DeepSeek 默认渠道-self-service"}},
|
||||
Accounts: []sub2api.NamedResource{{ID: "account_1", Name: "deepseek-01"}, {ID: "account_2", Name: "deepseek-02"}},
|
||||
}
|
||||
|
||||
result, err := NewReconcileService(store, host).Reconcile(context.Background(), ReconcileRequest{
|
||||
HostID: "host-1",
|
||||
HostBaseURL: "https://sub2api.example.com",
|
||||
AccessProbeAPIKey: "user-key",
|
||||
Pack: pack.LoadedPack{Manifest: pack.Manifest{PackID: "openai-cn-pack", Version: "1.0.0", TargetHost: "sub2api", MinHostVersion: "0.1.126", MaxHostVersion: "0.2.x"}},
|
||||
Provider: sampleProviderManifest(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Reconcile() error = %v", err)
|
||||
}
|
||||
if result.Status != "active" {
|
||||
t.Fatalf("Status = %q, want active after repair", result.Status)
|
||||
}
|
||||
if host.disableResponsesCalls != 1 {
|
||||
t.Fatalf("disable responses calls = %d, want 1", host.disableResponsesCalls)
|
||||
}
|
||||
if len(host.disabledResponsesAccountIDs) != 2 {
|
||||
t.Fatalf("disabled responses account ids = %v, want both accounts", host.disabledResponsesAccountIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileServiceReturnsDriftedWhenManagedResourceMissing(t *testing.T) {
|
||||
store := openProvisionTestStore(t)
|
||||
defer closeProvisionTestStore(t, store)
|
||||
|
||||
@@ -110,7 +110,7 @@ func (s *RuntimeImportService) Import(ctx context.Context, req RuntimeImportRequ
|
||||
}
|
||||
|
||||
includeManagedResources := importErr == nil || req.Mode != ImportModeStrict
|
||||
if persistErr := s.persistRuntimeArtifacts(ctx, batchID, hostRow.ID, req.Access.Mode, report, includeManagedResources); persistErr != nil {
|
||||
if persistErr := s.persistRuntimeArtifacts(ctx, batchID, hostRow.ID, req.Access, report, includeManagedResources); persistErr != nil {
|
||||
return RuntimeImportResult{}, persistErr
|
||||
}
|
||||
if err := s.store.ImportBatches().UpdateStatus(ctx, batchID, report.BatchStatus, report.AccessStatus); err != nil {
|
||||
@@ -167,7 +167,7 @@ func (s *RuntimeImportService) ensureProvider(ctx context.Context, packID int64,
|
||||
return s.store.Providers().GetByPackIDAndProviderID(ctx, packID, provider.ProviderID)
|
||||
}
|
||||
|
||||
func (s *RuntimeImportService) persistRuntimeArtifacts(ctx context.Context, batchID, hostID int64, accessMode string, report ImportReport, includeManagedResources bool) error {
|
||||
func (s *RuntimeImportService) persistRuntimeArtifacts(ctx context.Context, batchID, hostID int64, access AccessRequest, report ImportReport, includeManagedResources bool) error {
|
||||
for i, account := range report.Accounts {
|
||||
validationStatus := account.ValidationStatus()
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
@@ -222,6 +222,9 @@ func (s *RuntimeImportService) persistRuntimeArtifacts(ctx context.Context, batc
|
||||
}
|
||||
|
||||
accessPayload, err := json.Marshal(map[string]any{
|
||||
"probe_api_key": strings.TrimSpace(access.ProbeAPIKey),
|
||||
"subscription_users": subscriptionUserIDs(access.Subscriptions),
|
||||
"subscription_days": subscriptionDurationDays(access.Subscriptions),
|
||||
"status_code": report.Gateway.StatusCode,
|
||||
"ok": report.Gateway.OK,
|
||||
"has_expected_model": report.Gateway.HasExpectedModel,
|
||||
@@ -236,7 +239,7 @@ func (s *RuntimeImportService) persistRuntimeArtifacts(ctx context.Context, batc
|
||||
}
|
||||
if _, err := s.store.AccessClosures().Create(ctx, sqlite.AccessClosureRecord{
|
||||
BatchID: batchID,
|
||||
ClosureType: firstNonEmpty(strings.TrimSpace(accessMode), "unknown"),
|
||||
ClosureType: firstNonEmpty(strings.TrimSpace(access.Mode), "unknown"),
|
||||
Status: firstNonEmpty(report.AccessStatus, AccessStatusBroken),
|
||||
DetailsJSON: string(accessPayload),
|
||||
}); err != nil {
|
||||
@@ -245,6 +248,23 @@ func (s *RuntimeImportService) persistRuntimeArtifacts(ctx context.Context, batc
|
||||
return nil
|
||||
}
|
||||
|
||||
func subscriptionUserIDs(targets []SubscriptionTarget) []string {
|
||||
values := make([]string, 0, len(targets))
|
||||
for _, target := range targets {
|
||||
if trimmed := strings.TrimSpace(target.UserID); trimmed != "" {
|
||||
values = append(values, trimmed)
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func subscriptionDurationDays(targets []SubscriptionTarget) int {
|
||||
if len(targets) == 0 {
|
||||
return 0
|
||||
}
|
||||
return targets[0].DurationDays
|
||||
}
|
||||
|
||||
func (s *RuntimeImportService) persistManagedResourceIfAbsent(ctx context.Context, batchID, hostID int64, resourceType, hostResourceID, resourceName string) error {
|
||||
resourceType = strings.TrimSpace(resourceType)
|
||||
hostResourceID = strings.TrimSpace(hostResourceID)
|
||||
|
||||
@@ -3,6 +3,7 @@ package provision
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -352,6 +353,114 @@ func TestRuntimeImportServicePersistsPartialManagedResourcesOnAccessFailure(t *t
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeImportServicePersistsSelfServiceProbeKeyInAccessClosure(t *testing.T) {
|
||||
store := openProvisionTestStore(t)
|
||||
defer closeProvisionTestStore(t, store)
|
||||
|
||||
seedProvisionHost(t, store, "host-1", "https://sub2api.example.com")
|
||||
|
||||
host := &fakeHostAdapter{
|
||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "deepseek-01"}},
|
||||
testResults: map[string]sub2api.ProbeResult{
|
||||
"account_1": {OK: true, Status: "passed"},
|
||||
},
|
||||
models: map[string][]sub2api.AccountModel{
|
||||
"account_1": {{ID: "deepseek-chat"}},
|
||||
},
|
||||
gatewayResult: sub2api.GatewayAccessResult{OK: true, StatusCode: 200, HasExpectedModel: true, Models: []string{"deepseek-chat"}, CompletionOK: true, CompletionStatus: 200},
|
||||
}
|
||||
|
||||
result, err := NewRuntimeImportService(store, host).Import(context.Background(), RuntimeImportRequest{
|
||||
HostID: "host-1",
|
||||
HostBaseURL: "https://sub2api.example.com",
|
||||
Pack: pack.LoadedPack{
|
||||
Manifest: pack.Manifest{PackID: "openai-cn-pack", Version: "1.0.0", TargetHost: "sub2api", MinHostVersion: "0.1.126", MaxHostVersion: "0.2.x"},
|
||||
Checksum: "checksum-1",
|
||||
},
|
||||
Provider: sampleProviderManifest(),
|
||||
Mode: ImportModePartial,
|
||||
Keys: []string{"key-1"},
|
||||
Access: AccessRequest{
|
||||
Mode: AccessModeSelfService,
|
||||
ProbeAPIKey: "user-probe-key",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RuntimeImportService.Import() error = %v", err)
|
||||
}
|
||||
|
||||
closures, err := store.AccessClosures().GetByBatchID(context.Background(), result.BatchID)
|
||||
if err != nil {
|
||||
t.Fatalf("AccessClosures().GetByBatchID() error = %v", err)
|
||||
}
|
||||
if len(closures) != 1 {
|
||||
t.Fatalf("access closures = %d, want 1", len(closures))
|
||||
}
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal([]byte(closures[0].DetailsJSON), &payload); err != nil {
|
||||
t.Fatalf("decode access closure details: %v", err)
|
||||
}
|
||||
if got, _ := payload["probe_api_key"].(string); got != "user-probe-key" {
|
||||
t.Fatalf("probe_api_key = %q, want user-probe-key", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeImportServicePersistsSubscriptionMetadataInAccessClosure(t *testing.T) {
|
||||
store := openProvisionTestStore(t)
|
||||
defer closeProvisionTestStore(t, store)
|
||||
|
||||
seedProvisionHost(t, store, "host-1", "https://sub2api.example.com")
|
||||
|
||||
host := &fakeHostAdapter{
|
||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "deepseek-01"}},
|
||||
testResults: map[string]sub2api.ProbeResult{
|
||||
"account_1": {OK: true, Status: "passed"},
|
||||
},
|
||||
models: map[string][]sub2api.AccountModel{
|
||||
"account_1": {{ID: "deepseek-chat"}},
|
||||
},
|
||||
gatewayResult: sub2api.GatewayAccessResult{OK: true, StatusCode: 200, HasExpectedModel: true, Models: []string{"deepseek-chat"}, CompletionOK: true, CompletionStatus: 200},
|
||||
}
|
||||
|
||||
result, err := NewRuntimeImportService(store, host).Import(context.Background(), RuntimeImportRequest{
|
||||
HostID: "host-1",
|
||||
HostBaseURL: "https://sub2api.example.com",
|
||||
Pack: pack.LoadedPack{
|
||||
Manifest: pack.Manifest{PackID: "openai-cn-pack", Version: "1.0.0", TargetHost: "sub2api", MinHostVersion: "0.1.126", MaxHostVersion: "0.2.x"},
|
||||
Checksum: "checksum-1",
|
||||
},
|
||||
Provider: sampleProviderManifest(),
|
||||
Mode: ImportModePartial,
|
||||
Keys: []string{"key-1"},
|
||||
Access: AccessRequest{
|
||||
Mode: AccessModeSubscription,
|
||||
Subscriptions: []SubscriptionTarget{{UserID: "user-42", DurationDays: 30}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RuntimeImportService.Import() error = %v", err)
|
||||
}
|
||||
|
||||
closures, err := store.AccessClosures().GetByBatchID(context.Background(), result.BatchID)
|
||||
if err != nil {
|
||||
t.Fatalf("AccessClosures().GetByBatchID() error = %v", err)
|
||||
}
|
||||
if len(closures) != 1 {
|
||||
t.Fatalf("access closures = %d, want 1", len(closures))
|
||||
}
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal([]byte(closures[0].DetailsJSON), &payload); err != nil {
|
||||
t.Fatalf("decode access closure details: %v", err)
|
||||
}
|
||||
users, _ := payload["subscription_users"].([]any)
|
||||
if len(users) != 1 || users[0] != "user-42" {
|
||||
t.Fatalf("subscription_users = %#v, want [user-42]", users)
|
||||
}
|
||||
if got, _ := payload["subscription_days"].(float64); int(got) != 30 {
|
||||
t.Fatalf("subscription_days = %v, want 30", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeImportServiceRepeatedImportReusesManagedResources(t *testing.T) {
|
||||
store := openProvisionTestStore(t)
|
||||
defer closeProvisionTestStore(t, store)
|
||||
|
||||
@@ -1,66 +1,37 @@
|
||||
package provision
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/access"
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
"sub2api-cn-relay-manager/internal/pack"
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
)
|
||||
|
||||
type BatchDetailResult struct {
|
||||
Batch sqlite.ImportBatch
|
||||
Items []sqlite.ImportBatchItem
|
||||
ManagedResources []sqlite.ManagedResource
|
||||
AccessClosures []sqlite.AccessClosureRecord
|
||||
ReconcileRuns []sqlite.ReconcileRun
|
||||
}
|
||||
const (
|
||||
accessModeSubscription = "subscription"
|
||||
accessModeSelfService = "self_service"
|
||||
|
||||
type BatchDetailService struct {
|
||||
store *sqlite.DB
|
||||
}
|
||||
accessStatusSubscriptionReady = "subscription_ready"
|
||||
accessStatusSelfServiceReady = "self_service_ready"
|
||||
accessStatusBroken = "broken"
|
||||
|
||||
func NewBatchDetailService(store *sqlite.DB) *BatchDetailService {
|
||||
return &BatchDetailService{store: store}
|
||||
}
|
||||
batchStatusSucceeded = "succeeded"
|
||||
batchStatusPartial = "partially_succeeded"
|
||||
|
||||
func (s *BatchDetailService) Get(ctx context.Context, batchID int64) (BatchDetailResult, error) {
|
||||
if s == nil || s.store == nil {
|
||||
return BatchDetailResult{}, fmt.Errorf("store is required")
|
||||
}
|
||||
batch, err := s.store.ImportBatches().GetByID(ctx, batchID)
|
||||
if err != nil {
|
||||
return BatchDetailResult{}, err
|
||||
}
|
||||
items, err := s.store.ImportBatchItems().GetByBatchID(ctx, batchID)
|
||||
if err != nil {
|
||||
return BatchDetailResult{}, err
|
||||
}
|
||||
managedResources, err := s.store.ManagedResources().GetByBatchID(ctx, batchID)
|
||||
if err != nil {
|
||||
return BatchDetailResult{}, err
|
||||
}
|
||||
accessClosures, err := s.store.AccessClosures().GetByBatchID(ctx, batchID)
|
||||
if err != nil {
|
||||
return BatchDetailResult{}, err
|
||||
}
|
||||
reconcileRuns, err := s.store.ReconcileRuns().GetByBatchID(ctx, batchID)
|
||||
if err != nil {
|
||||
return BatchDetailResult{}, err
|
||||
}
|
||||
return BatchDetailResult{
|
||||
Batch: batch,
|
||||
Items: items,
|
||||
ManagedResources: managedResources,
|
||||
AccessClosures: accessClosures,
|
||||
ReconcileRuns: reconcileRuns,
|
||||
}, nil
|
||||
}
|
||||
accountStatusPassed = "passed"
|
||||
accountStatusWarning = "warning"
|
||||
accountStatusFailed = "failed"
|
||||
)
|
||||
|
||||
type ReconcileRequest struct {
|
||||
var nonSlugPattern = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
type Request struct {
|
||||
HostID string
|
||||
HostBaseURL string
|
||||
AccessProbeAPIKey string
|
||||
@@ -68,7 +39,7 @@ type ReconcileRequest struct {
|
||||
Provider pack.ProviderManifest
|
||||
}
|
||||
|
||||
type ReconcileResult struct {
|
||||
type Result struct {
|
||||
BatchID int64
|
||||
Status string
|
||||
MissingCount int
|
||||
@@ -79,97 +50,103 @@ type ReconcileResult struct {
|
||||
Summary map[string]any
|
||||
}
|
||||
|
||||
type ReconcileService struct {
|
||||
type accountProbeRerunSummary struct {
|
||||
Failures int
|
||||
AccountIDs []string
|
||||
ResponsesCapabilitySuspect bool
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
store *sqlite.DB
|
||||
host sub2api.HostAdapter
|
||||
}
|
||||
|
||||
func NewReconcileService(store *sqlite.DB, host sub2api.HostAdapter) *ReconcileService {
|
||||
return &ReconcileService{store: store, host: host}
|
||||
func NewService(store *sqlite.DB, host sub2api.HostAdapter) *Service {
|
||||
return &Service{store: store, host: host}
|
||||
}
|
||||
|
||||
func (s *ReconcileService) Reconcile(ctx context.Context, req ReconcileRequest) (ReconcileResult, error) {
|
||||
func (s *Service) Reconcile(ctx context.Context, req Request) (Result, error) {
|
||||
if s == nil || s.store == nil {
|
||||
return ReconcileResult{}, fmt.Errorf("store is required")
|
||||
return Result{}, fmt.Errorf("store is required")
|
||||
}
|
||||
if s.host == nil {
|
||||
return ReconcileResult{}, fmt.Errorf("host adapter is required")
|
||||
return Result{}, fmt.Errorf("host adapter is required")
|
||||
}
|
||||
if strings.TrimSpace(req.HostID) == "" {
|
||||
return ReconcileResult{}, fmt.Errorf("host_id is required")
|
||||
return Result{}, fmt.Errorf("host_id is required")
|
||||
}
|
||||
if strings.TrimSpace(req.HostBaseURL) == "" {
|
||||
return ReconcileResult{}, fmt.Errorf("host_base_url is required")
|
||||
return Result{}, fmt.Errorf("host_base_url is required")
|
||||
}
|
||||
hostVersion, err := s.host.GetHostVersion(ctx)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, fmt.Errorf("get host version: %w", err)
|
||||
return Result{}, fmt.Errorf("get host version: %w", err)
|
||||
}
|
||||
if err := pack.CheckHostCompatibility(req.Pack.Manifest, hostVersion); err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
packRow, err := s.store.Packs().GetByPackID(ctx, req.Pack.Manifest.PackID)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
providerRow, err := s.store.Providers().GetByPackIDAndProviderID(ctx, packRow.ID, req.Provider.ProviderID)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
hostRow, err := s.store.Hosts().GetByHostID(ctx, req.HostID)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
batchRow, err := s.store.ImportBatches().GetLatestByProviderIDAndHostID(ctx, providerRow.ID, hostRow.ID)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
switch strings.TrimSpace(batchRow.BatchStatus) {
|
||||
case BatchStatusSucceeded, BatchStatusPartial:
|
||||
case batchStatusSucceeded, batchStatusPartial:
|
||||
default:
|
||||
return ReconcileResult{}, fmt.Errorf("latest import batch is %s; run import again before reconcile", batchRow.BatchStatus)
|
||||
return Result{}, fmt.Errorf("latest import batch is %s; run import again before reconcile", batchRow.BatchStatus)
|
||||
}
|
||||
storedResources, err := s.storedResourcesForReconcile(ctx, providerRow.ID, hostRow.ID, batchRow.ID)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
currentBatchResources, err := s.store.ManagedResources().GetByBatchID(ctx, batchRow.ID)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
batchItems, err := s.store.ImportBatchItems().GetByBatchID(ctx, batchRow.ID)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
accessClosures, err := s.store.AccessClosures().GetByBatchID(ctx, batchRow.ID)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
snapshot, err := s.host.ListManagedResources(ctx, buildManagedResourceListRequest(req.Provider, accessClosureType(accessClosures)))
|
||||
if err != nil {
|
||||
return ReconcileResult{}, fmt.Errorf("list managed resources: %w", err)
|
||||
return Result{}, fmt.Errorf("list managed resources: %w", err)
|
||||
}
|
||||
missing, extra := diffManagedResources(storedResources, snapshot)
|
||||
rawExtra := extra
|
||||
staleNoiseAccounts := classifyHistoricalAccountNoise(currentBatchResources, snapshot.Accounts, SuggestAccountNamePrefix(req.Provider))
|
||||
staleNoiseAccounts := classifyHistoricalAccountNoise(currentBatchResources, snapshot.Accounts, suggestAccountNamePrefix(req.Provider))
|
||||
if len(staleNoiseAccounts) > 0 {
|
||||
extra -= len(staleNoiseAccounts)
|
||||
if extra < 0 {
|
||||
extra = 0
|
||||
}
|
||||
}
|
||||
probeFailures, err := s.rerunAccountProbes(ctx, batchItems, req.Provider.SmokeTestModel)
|
||||
probeSummary, err := s.rerunAccountProbes(ctx, batchItems, req.Provider.SmokeTestModel)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
accessStatus, accessChecked, err := s.rerunAccessClosure(ctx, batchRow.ID, accessClosures, req.AccessProbeAPIKey, req.Provider.SmokeTestModel)
|
||||
accessStatus, accessChecked, err := s.rerunAccessClosure(ctx, batchRow.ID, accessClosures, req.AccessProbeAPIKey, req.Provider.SmokeTestModel, probeSummary.AccountIDs, probeSummary.ResponsesCapabilitySuspect)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
status := "active"
|
||||
if missing > 0 || extra > 0 {
|
||||
status = "drifted"
|
||||
} else if probeFailures > 0 || (accessChecked && accessStatus == AccessStatusBroken) {
|
||||
} else if probeSummary.Failures > 0 || (accessChecked && accessStatus == accessStatusBroken) {
|
||||
status = "degraded"
|
||||
}
|
||||
summary := map[string]any{
|
||||
@@ -179,48 +156,56 @@ func (s *ReconcileService) Reconcile(ctx context.Context, req ReconcileRequest)
|
||||
"stale_noise_count": len(staleNoiseAccounts),
|
||||
"stale_noise_accounts": staleNoiseAccounts,
|
||||
"host_version": hostVersion,
|
||||
"probe_failures": probeFailures,
|
||||
"probe_failures": probeSummary.Failures,
|
||||
"access_status": accessStatus,
|
||||
"access_rechecked": accessChecked,
|
||||
}
|
||||
summaryJSON, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
return ReconcileResult{}, fmt.Errorf("marshal reconcile summary: %w", err)
|
||||
return Result{}, fmt.Errorf("marshal reconcile summary: %w", err)
|
||||
}
|
||||
if _, err := s.store.ReconcileRuns().Create(ctx, sqlite.ReconcileRun{BatchID: batchRow.ID, HostID: hostRow.ID, ProviderID: providerRow.ID, Status: status, SummaryJSON: string(summaryJSON)}); err != nil {
|
||||
return ReconcileResult{}, err
|
||||
return Result{}, err
|
||||
}
|
||||
return ReconcileResult{BatchID: batchRow.ID, Status: status, MissingCount: missing, ExtraCount: extra, StaleNoiseCount: len(staleNoiseAccounts), ProbeFailureCount: probeFailures, AccessStatus: accessStatus, Summary: summary}, nil
|
||||
return Result{
|
||||
BatchID: batchRow.ID,
|
||||
Status: status,
|
||||
MissingCount: missing,
|
||||
ExtraCount: extra,
|
||||
StaleNoiseCount: len(staleNoiseAccounts),
|
||||
ProbeFailureCount: probeSummary.Failures,
|
||||
AccessStatus: accessStatus,
|
||||
Summary: summary,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ReconcileService) rerunAccountProbes(ctx context.Context, items []sqlite.ImportBatchItem, expectedModel string) (int, error) {
|
||||
func (s *Service) rerunAccountProbes(ctx context.Context, items []sqlite.ImportBatchItem, expectedModel string) (accountProbeRerunSummary, error) {
|
||||
if len(items) == 0 {
|
||||
return 0, nil
|
||||
return accountProbeRerunSummary{}, nil
|
||||
}
|
||||
failures := 0
|
||||
summary := accountProbeRerunSummary{}
|
||||
for _, item := range items {
|
||||
accountID, err := accountIDFromProbeSummary(item.ProbeSummaryJSON)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("decode import batch item %d probe summary: %w", item.ID, err)
|
||||
return accountProbeRerunSummary{}, fmt.Errorf("decode import batch item %d probe summary: %w", item.ID, err)
|
||||
}
|
||||
if strings.TrimSpace(accountID) == "" {
|
||||
return 0, fmt.Errorf("import batch item %d missing account_id in probe summary", item.ID)
|
||||
return accountProbeRerunSummary{}, fmt.Errorf("import batch item %d missing account_id in probe summary", item.ID)
|
||||
}
|
||||
summary.AccountIDs = append(summary.AccountIDs, accountID)
|
||||
probe, err := s.host.TestAccount(ctx, accountID, expectedModel)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("re-test account %s: %w", accountID, err)
|
||||
return accountProbeRerunSummary{}, fmt.Errorf("re-test account %s: %w", accountID, err)
|
||||
}
|
||||
models, err := s.host.GetAccountModels(ctx, accountID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("reload account models %s: %w", accountID, err)
|
||||
return accountProbeRerunSummary{}, fmt.Errorf("reload account models %s: %w", accountID, err)
|
||||
}
|
||||
smokeModelSeen := hasModel(models, expectedModel)
|
||||
result := AccountImportResult{
|
||||
Probe: probe,
|
||||
Models: models,
|
||||
SmokeModelSeen: smokeModelSeen,
|
||||
status := accountValidationStatus(probe, smokeModelSeen)
|
||||
if smokeModelSeen && access.SuspectsOpenAIResponsesCapabilityMismatch(probe) {
|
||||
summary.ResponsesCapabilitySuspect = true
|
||||
}
|
||||
status := result.ValidationStatus()
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"account_id": accountID,
|
||||
"probe_ok": probe.OK,
|
||||
@@ -228,27 +213,28 @@ func (s *ReconcileService) rerunAccountProbes(ctx context.Context, items []sqlit
|
||||
"probe_message": probe.Message,
|
||||
"models": models,
|
||||
"smoke_model_seen": smokeModelSeen,
|
||||
"probe_advisory": result.HasAdvisoryWarning(),
|
||||
"probe_advisory": status == accountStatusWarning,
|
||||
"validation_status": status,
|
||||
"reconcile_rerun": true,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("marshal probe rerun summary for %s: %w", accountID, err)
|
||||
return accountProbeRerunSummary{}, fmt.Errorf("marshal probe rerun summary for %s: %w", accountID, err)
|
||||
}
|
||||
if err := s.store.ImportBatchItems().UpdateResult(ctx, item.ID, status, string(payload)); err != nil {
|
||||
return 0, err
|
||||
return accountProbeRerunSummary{}, err
|
||||
}
|
||||
if _, err := s.store.ProbeResults().Create(ctx, sqlite.ProbeResult{BatchItemID: item.ID, ProbeType: "account_smoke_rerun", Status: status, SummaryJSON: string(payload)}); err != nil {
|
||||
return 0, err
|
||||
return accountProbeRerunSummary{}, err
|
||||
}
|
||||
if result.HasBlockingFailure() {
|
||||
failures++
|
||||
if status == accountStatusFailed {
|
||||
summary.Failures++
|
||||
}
|
||||
}
|
||||
return failures, nil
|
||||
summary.AccountIDs = normalizedUniqueAccountIDs(summary.AccountIDs)
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (s *ReconcileService) rerunAccessClosure(ctx context.Context, batchID int64, accessClosures []sqlite.AccessClosureRecord, probeAPIKey, expectedModel string) (string, bool, error) {
|
||||
func (s *Service) rerunAccessClosure(ctx context.Context, batchID int64, accessClosures []sqlite.AccessClosureRecord, probeAPIKey, expectedModel string, accountIDs []string, suspectResponsesCapabilityMismatch bool) (string, bool, error) {
|
||||
if len(accessClosures) == 0 {
|
||||
return "not_configured", false, nil
|
||||
}
|
||||
@@ -262,24 +248,33 @@ func (s *ReconcileService) rerunAccessClosure(ctx context.Context, batchID int64
|
||||
return "", false, fmt.Errorf("re-check gateway access: %w", err)
|
||||
}
|
||||
if result.OK && result.HasExpectedModel {
|
||||
completion, err := s.host.CheckGatewayCompletion(ctx, sub2api.GatewayCompletionCheckRequest{
|
||||
completionReq := sub2api.GatewayCompletionCheckRequest{
|
||||
APIKey: probeAPIKey,
|
||||
Model: expectedModel,
|
||||
Prompt: "ping",
|
||||
MaxTokens: 8,
|
||||
})
|
||||
}
|
||||
completion, err := s.host.CheckGatewayCompletion(ctx, completionReq)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("re-check gateway completion: %w", err)
|
||||
}
|
||||
if access.ShouldAttemptOpenAIResponsesCapabilityRepair(suspectResponsesCapabilityMismatch, completion) {
|
||||
if err := s.host.DisableOpenAIResponsesAPI(ctx, accountIDs); err == nil {
|
||||
completion, err = s.host.CheckGatewayCompletion(ctx, completionReq)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("re-check gateway completion after capability repair: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
result.CompletionOK = completion.OK
|
||||
result.CompletionStatus = completion.StatusCode
|
||||
result.CompletionType = completion.ContentType
|
||||
result.CompletionBody = completion.BodyPreview
|
||||
}
|
||||
if GatewayAccessReady(result) {
|
||||
if gatewayAccessReady(result) {
|
||||
status = deriveHealthyAccessStatus(latest.ClosureType)
|
||||
} else {
|
||||
status = AccessStatusBroken
|
||||
status = accessStatusBroken
|
||||
}
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"status_code": result.StatusCode,
|
||||
@@ -301,17 +296,38 @@ func (s *ReconcileService) rerunAccessClosure(ctx context.Context, batchID int64
|
||||
return status, true, nil
|
||||
}
|
||||
|
||||
func normalizedUniqueAccountIDs(accountIDs []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
values := make([]string, 0, len(accountIDs))
|
||||
for _, rawID := range accountIDs {
|
||||
accountID := strings.TrimSpace(rawID)
|
||||
if accountID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[accountID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[accountID] = struct{}{}
|
||||
values = append(values, accountID)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func deriveHealthyAccessStatus(closureType string) string {
|
||||
switch strings.TrimSpace(closureType) {
|
||||
case AccessModeSubscription:
|
||||
return AccessStatusSubscriptionReady
|
||||
case AccessModeSelfService:
|
||||
return AccessStatusSelfServiceReady
|
||||
case accessModeSubscription:
|
||||
return accessStatusSubscriptionReady
|
||||
case accessModeSelfService:
|
||||
return accessStatusSelfServiceReady
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func DeriveHealthyAccessStatus(closureType string) string {
|
||||
return deriveHealthyAccessStatus(closureType)
|
||||
}
|
||||
|
||||
func accessClosureType(accessClosures []sqlite.AccessClosureRecord) string {
|
||||
if len(accessClosures) == 0 {
|
||||
return ""
|
||||
@@ -320,13 +336,13 @@ func accessClosureType(accessClosures []sqlite.AccessClosureRecord) string {
|
||||
}
|
||||
|
||||
func buildManagedResourceListRequest(provider pack.ProviderManifest, accessMode string) sub2api.ListManagedResourcesRequest {
|
||||
names := SuggestResourceNamesForMode(provider, accessMode)
|
||||
names := suggestResourceNamesForMode(provider, accessMode)
|
||||
req := sub2api.ListManagedResourcesRequest{
|
||||
GroupName: names.Group,
|
||||
ChannelName: names.Channel,
|
||||
AccountNamePrefix: SuggestAccountNamePrefix(provider),
|
||||
AccountNamePrefix: suggestAccountNamePrefix(provider),
|
||||
}
|
||||
if strings.TrimSpace(accessMode) == AccessModeSubscription {
|
||||
if strings.TrimSpace(accessMode) == accessModeSubscription {
|
||||
req.PlanName = names.Plan
|
||||
}
|
||||
return req
|
||||
@@ -344,7 +360,11 @@ func accountIDFromProbeSummary(summaryJSON string) (string, error) {
|
||||
return strings.TrimSpace(accountID), nil
|
||||
}
|
||||
|
||||
func (s *ReconcileService) storedResourcesForReconcile(ctx context.Context, providerID, hostID, batchID int64) ([]sqlite.ManagedResource, error) {
|
||||
func AccountIDFromProbeSummary(summaryJSON string) (string, error) {
|
||||
return accountIDFromProbeSummary(summaryJSON)
|
||||
}
|
||||
|
||||
func (s *Service) storedResourcesForReconcile(ctx context.Context, providerID, hostID, batchID int64) ([]sqlite.ManagedResource, error) {
|
||||
storedResources, err := s.store.ManagedResources().GetByBatchID(ctx, batchID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -427,6 +447,10 @@ func diffManagedResources(stored []sqlite.ManagedResource, snapshot sub2api.Mana
|
||||
return missing, extra
|
||||
}
|
||||
|
||||
func DiffManagedResources(stored []sqlite.ManagedResource, snapshot sub2api.ManagedResourceSnapshot) (int, int) {
|
||||
return diffManagedResources(stored, snapshot)
|
||||
}
|
||||
|
||||
func classifyHistoricalAccountNoise(currentBatchResources []sqlite.ManagedResource, snapshotAccounts []sub2api.NamedResource, accountNamePrefix string) []sub2api.NamedResource {
|
||||
currentAccountIDs := make(map[string]struct{})
|
||||
for _, resource := range currentBatchResources {
|
||||
@@ -455,3 +479,160 @@ func classifyHistoricalAccountNoise(currentBatchResources []sqlite.ManagedResour
|
||||
}
|
||||
return staleNoise
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func hasModel(models []sub2api.AccountModel, target string) bool {
|
||||
for _, model := range models {
|
||||
if strings.TrimSpace(model.ID) == strings.TrimSpace(target) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func gatewayAccessReady(result sub2api.GatewayAccessResult) bool {
|
||||
return result.OK && result.HasExpectedModel && result.CompletionOK
|
||||
}
|
||||
|
||||
func accountValidationStatus(probe sub2api.ProbeResult, smokeModelSeen bool) string {
|
||||
if !smokeModelSeen {
|
||||
return accountStatusFailed
|
||||
}
|
||||
if probe.OK {
|
||||
return accountStatusPassed
|
||||
}
|
||||
if isAdvisoryAccountProbeFailure(probe) {
|
||||
return accountStatusWarning
|
||||
}
|
||||
return accountStatusFailed
|
||||
}
|
||||
|
||||
func isAdvisoryAccountProbeFailure(probe sub2api.ProbeResult) bool {
|
||||
if probe.OK {
|
||||
return false
|
||||
}
|
||||
|
||||
message := strings.ToLower(strings.TrimSpace(probe.Message))
|
||||
if message == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
if isTransientAccountProbeFailure(message) {
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.Contains(message, "api returned 403: forbidden") {
|
||||
return true
|
||||
}
|
||||
|
||||
if !strings.Contains(message, "responses api") {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.Contains(message, "当前测试接口仅支持") ||
|
||||
strings.Contains(message, "账号本身可正常使用") ||
|
||||
strings.Contains(message, "please directly") ||
|
||||
strings.Contains(message, "actual api")
|
||||
}
|
||||
|
||||
func isTransientAccountProbeFailure(message string) bool {
|
||||
if !(strings.Contains(message, "429") ||
|
||||
strings.Contains(message, "rate limit") ||
|
||||
strings.Contains(message, "too many requests") ||
|
||||
strings.Contains(message, "502") ||
|
||||
strings.Contains(message, "503") ||
|
||||
strings.Contains(message, "504") ||
|
||||
strings.Contains(message, "bad gateway") ||
|
||||
strings.Contains(message, "service unavailable") ||
|
||||
strings.Contains(message, "timeout")) {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.Contains(message, "api returned") ||
|
||||
strings.Contains(message, "rate_limit") ||
|
||||
strings.Contains(message, "upstream") ||
|
||||
strings.Contains(message, "temporar") ||
|
||||
strings.Contains(message, "retry")
|
||||
}
|
||||
|
||||
type resourceNames struct {
|
||||
Group string
|
||||
Channel string
|
||||
Plan string
|
||||
}
|
||||
|
||||
func suggestAccountNamePrefix(provider pack.ProviderManifest) string {
|
||||
return fmt.Sprintf("%s-", resourceSlug(provider.ProviderID))
|
||||
}
|
||||
|
||||
func suggestResourceNames(provider pack.ProviderManifest) resourceNames {
|
||||
slug := resourceSlug(provider.ProviderID)
|
||||
return resourceNames{
|
||||
Group: fallbackString(strings.TrimSpace(provider.GroupTemplate.Name), fmt.Sprintf("crm-%s-group", slug)),
|
||||
Channel: fallbackString(strings.TrimSpace(provider.ChannelTemplate.Name), fmt.Sprintf("crm-%s-channel", slug)),
|
||||
Plan: fallbackString(strings.TrimSpace(provider.PlanTemplate.Name), fmt.Sprintf("crm-%s-plan", slug)),
|
||||
}
|
||||
}
|
||||
|
||||
func suggestResourceNamesForMode(provider pack.ProviderManifest, accessMode string) resourceNames {
|
||||
base := suggestResourceNames(provider)
|
||||
suffix := accessModeResourceSuffix(accessMode)
|
||||
if suffix == "" {
|
||||
return base
|
||||
}
|
||||
return resourceNames{
|
||||
Group: appendResourceNameSuffix(base.Group, suffix),
|
||||
Channel: appendResourceNameSuffix(base.Channel, suffix),
|
||||
Plan: appendResourceNameSuffix(base.Plan, suffix),
|
||||
}
|
||||
}
|
||||
|
||||
func accessModeResourceSuffix(accessMode string) string {
|
||||
switch strings.TrimSpace(accessMode) {
|
||||
case accessModeSubscription:
|
||||
return "subscription"
|
||||
case accessModeSelfService:
|
||||
return "self-service"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func appendResourceNameSuffix(name, suffix string) string {
|
||||
name = strings.TrimSpace(name)
|
||||
suffix = strings.TrimSpace(suffix)
|
||||
if name == "" || suffix == "" {
|
||||
return name
|
||||
}
|
||||
if strings.HasSuffix(name, "-"+suffix) {
|
||||
return name
|
||||
}
|
||||
return name + "-" + suffix
|
||||
}
|
||||
|
||||
func resourceSlug(raw string) string {
|
||||
slug := strings.ToLower(strings.TrimSpace(raw))
|
||||
slug = nonSlugPattern.ReplaceAllString(slug, "-")
|
||||
slug = strings.Trim(slug, "-")
|
||||
if slug == "" {
|
||||
return "provider"
|
||||
}
|
||||
return slug
|
||||
}
|
||||
|
||||
func fallbackString(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
104
internal/reconcile/service_test.go
Normal file
104
internal/reconcile/service_test.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
"sub2api-cn-relay-manager/internal/pack"
|
||||
"sub2api-cn-relay-manager/internal/store/sqlite"
|
||||
)
|
||||
|
||||
func TestAccountIDFromProbeSummary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
accountID, err := AccountIDFromProbeSummary(`{"account_id":" account_1 "}`)
|
||||
if err != nil {
|
||||
t.Fatalf("AccountIDFromProbeSummary() error = %v", err)
|
||||
}
|
||||
if accountID != "account_1" {
|
||||
t.Fatalf("accountID = %q, want account_1", accountID)
|
||||
}
|
||||
if _, err := AccountIDFromProbeSummary(`{`); err == nil {
|
||||
t.Fatal("AccountIDFromProbeSummary() error = nil, want JSON decode error")
|
||||
}
|
||||
blank, err := AccountIDFromProbeSummary("")
|
||||
if err != nil {
|
||||
t.Fatalf("AccountIDFromProbeSummary(blank) error = %v", err)
|
||||
}
|
||||
if blank != "" {
|
||||
t.Fatalf("blank accountID = %q, want empty", blank)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffManagedResourcesCountsMissingAndExtra(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
missing, extra := DiffManagedResources(
|
||||
[]sqlite.ManagedResource{
|
||||
{ResourceType: "group", HostResourceID: "group_1"},
|
||||
{ResourceType: "account", HostResourceID: "account_1"},
|
||||
},
|
||||
sub2api.ManagedResourceSnapshot{
|
||||
Groups: []sub2api.NamedResource{{ID: "group_1"}},
|
||||
Accounts: []sub2api.NamedResource{{ID: "account_2"}},
|
||||
},
|
||||
)
|
||||
if missing != 1 || extra != 1 {
|
||||
t.Fatalf("DiffManagedResources() = (%d, %d), want (1, 1)", missing, extra)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveHealthyAccessStatus(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
closureType string
|
||||
want string
|
||||
}{
|
||||
{name: "subscription", closureType: accessModeSubscription, want: accessStatusSubscriptionReady},
|
||||
{name: "self-service", closureType: accessModeSelfService, want: accessStatusSelfServiceReady},
|
||||
{name: "unknown", closureType: "other", want: "unknown"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := DeriveHealthyAccessStatus(tc.closureType); got != tc.want {
|
||||
t.Fatalf("DeriveHealthyAccessStatus(%q) = %q, want %q", tc.closureType, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildManagedResourceListRequestUsesModeSpecificNames(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
provider := pack.ProviderManifest{ProviderID: "deepseek"}
|
||||
subscriptionReq := buildManagedResourceListRequest(provider, accessModeSubscription)
|
||||
if subscriptionReq.PlanName == "" {
|
||||
t.Fatal("subscription PlanName = empty, want mode-specific plan")
|
||||
}
|
||||
if subscriptionReq.AccountNamePrefix != "deepseek-" {
|
||||
t.Fatalf("subscription AccountNamePrefix = %q, want deepseek-", subscriptionReq.AccountNamePrefix)
|
||||
}
|
||||
|
||||
selfServiceReq := buildManagedResourceListRequest(provider, accessModeSelfService)
|
||||
if selfServiceReq.PlanName != "" {
|
||||
t.Fatalf("selfService PlanName = %q, want empty", selfServiceReq.PlanName)
|
||||
}
|
||||
if selfServiceReq.GroupName == subscriptionReq.GroupName {
|
||||
t.Fatalf("GroupName = %q, want different names per access mode", selfServiceReq.GroupName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountValidationStatusTreatsResponsesRaceAsWarning(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
status := accountValidationStatus(sub2api.ProbeResult{
|
||||
OK: false,
|
||||
Status: "failed",
|
||||
Message: "账号本身可正常使用,但当前测试接口仅支持 Responses API 路径。请直接通过实际 API 调用验证。",
|
||||
}, true)
|
||||
if status != accountStatusWarning {
|
||||
t.Fatalf("accountValidationStatus() = %q, want %q", status, accountStatusWarning)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
ALTER TABLE import_runs ADD COLUMN host_id TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE import_runs ADD COLUMN subscription_users_json TEXT NOT NULL DEFAULT '[]';
|
||||
ALTER TABLE import_runs ADD COLUMN subscription_days INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE import_runs ADD COLUMN probe_api_key TEXT NOT NULL DEFAULT '';
|
||||
|
||||
CREATE INDEX idx_import_runs_host_id ON import_runs(host_id);
|
||||
@@ -182,6 +182,36 @@ func (r *ImportBatchesRepo) ListByProviderIDAndHostID(ctx context.Context, provi
|
||||
return batches, nil
|
||||
}
|
||||
|
||||
func (r *ImportBatchesRepo) ListLatestReconcilable(ctx context.Context) ([]ImportBatch, error) {
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
SELECT ib.id, ib.host_id, ib.pack_id, ib.provider_id, ib.mode, ib.batch_status, ib.access_status
|
||||
FROM import_batches ib
|
||||
INNER JOIN (
|
||||
SELECT provider_id, host_id, MAX(id) AS latest_id
|
||||
FROM import_batches
|
||||
GROUP BY provider_id, host_id
|
||||
) latest ON latest.latest_id = ib.id
|
||||
WHERE ib.batch_status IN ('succeeded', 'partially_succeeded')
|
||||
ORDER BY ib.id DESC`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query latest reconcilable import batches: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
batches := make([]ImportBatch, 0)
|
||||
for rows.Next() {
|
||||
var batch ImportBatch
|
||||
if err := rows.Scan(&batch.ID, &batch.HostID, &batch.PackID, &batch.ProviderID, &batch.Mode, &batch.BatchStatus, &batch.AccessStatus); err != nil {
|
||||
return nil, fmt.Errorf("scan latest reconcilable import batch: %w", err)
|
||||
}
|
||||
batches = append(batches, batch)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate latest reconcilable import batches: %w", err)
|
||||
}
|
||||
return batches, nil
|
||||
}
|
||||
|
||||
func (r *ImportBatchItemsRepo) GetByBatchID(ctx context.Context, batchID int64) ([]ImportBatchItem, error) {
|
||||
if batchID <= 0 {
|
||||
return nil, fmt.Errorf("batch_id is required")
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ImportRunItem struct {
|
||||
@@ -236,6 +237,47 @@ func (r *ImportRunItemsRepo) ListByRunID(ctx context.Context, runID string) ([]I
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (r *ImportRunItemsRepo) TryAcquireConfirmationLease(ctx context.Context, itemID, workerID string, now time.Time, leaseDuration time.Duration) (ImportRunItem, bool, error) {
|
||||
itemID = strings.TrimSpace(itemID)
|
||||
workerID = strings.TrimSpace(workerID)
|
||||
if itemID == "" {
|
||||
return ImportRunItem{}, false, fmt.Errorf("item_id is required")
|
||||
}
|
||||
if workerID == "" {
|
||||
return ImportRunItem{}, false, fmt.Errorf("worker_id is required")
|
||||
}
|
||||
if leaseDuration <= 0 {
|
||||
leaseDuration = time.Minute
|
||||
}
|
||||
|
||||
nowText := now.UTC().Format(time.RFC3339)
|
||||
leaseUntil := now.UTC().Add(leaseDuration).Format(time.RFC3339)
|
||||
result, err := r.db.ExecContext(ctx, `UPDATE import_run_items
|
||||
SET lease_owner = ?, lease_until = ?, confirmation_attempts = confirmation_attempts + 1, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE item_id = ?
|
||||
AND current_stage = 'confirm'
|
||||
AND confirmation_status = 'pending'
|
||||
AND (next_retry_at IS NULL OR next_retry_at = '' OR next_retry_at <= ?)
|
||||
AND (lease_until IS NULL OR lease_until = '' OR lease_until < ?)`,
|
||||
workerID, leaseUntil, itemID, nowText, nowText)
|
||||
if err != nil {
|
||||
return ImportRunItem{}, false, fmt.Errorf("acquire confirmation lease for %q: %w", itemID, err)
|
||||
}
|
||||
rows, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return ImportRunItem{}, false, err
|
||||
}
|
||||
if rows == 0 {
|
||||
return ImportRunItem{}, false, nil
|
||||
}
|
||||
|
||||
item, err := r.GetByItemID(ctx, itemID)
|
||||
if err != nil {
|
||||
return ImportRunItem{}, false, err
|
||||
}
|
||||
return item, true, nil
|
||||
}
|
||||
|
||||
func boolToInt(value bool) int {
|
||||
if value {
|
||||
return 1
|
||||
|
||||
@@ -7,19 +7,23 @@ import (
|
||||
)
|
||||
|
||||
type ImportRun struct {
|
||||
RunID string
|
||||
Mode string
|
||||
AccessMode string
|
||||
State string
|
||||
TotalItems int
|
||||
CompletedItems int
|
||||
ActiveItems int
|
||||
DegradedItems int
|
||||
BrokenItems int
|
||||
WarningItems int
|
||||
StartedAt string
|
||||
UpdatedAt string
|
||||
FinishedAt string
|
||||
RunID string
|
||||
HostID string
|
||||
Mode string
|
||||
AccessMode string
|
||||
SubscriptionUsersJSON string
|
||||
SubscriptionDays int
|
||||
ProbeAPIKey string
|
||||
State string
|
||||
TotalItems int
|
||||
CompletedItems int
|
||||
ActiveItems int
|
||||
DegradedItems int
|
||||
BrokenItems int
|
||||
WarningItems int
|
||||
StartedAt string
|
||||
UpdatedAt string
|
||||
FinishedAt string
|
||||
}
|
||||
|
||||
type ImportRunsRepo struct {
|
||||
@@ -32,13 +36,18 @@ func newImportRunsRepo(db execQuerier) *ImportRunsRepo {
|
||||
|
||||
func (r *ImportRunsRepo) Create(ctx context.Context, run ImportRun) error {
|
||||
runID := strings.TrimSpace(run.RunID)
|
||||
hostID := strings.TrimSpace(run.HostID)
|
||||
mode := strings.TrimSpace(run.Mode)
|
||||
accessMode := strings.TrimSpace(run.AccessMode)
|
||||
subscriptionUsersJSON := defaultJSON(run.SubscriptionUsersJSON, "[]")
|
||||
probeAPIKey := strings.TrimSpace(run.ProbeAPIKey)
|
||||
state := strings.TrimSpace(run.State)
|
||||
|
||||
switch {
|
||||
case runID == "":
|
||||
return fmt.Errorf("run_id is required")
|
||||
case hostID == "":
|
||||
return fmt.Errorf("host_id is required")
|
||||
case mode == "":
|
||||
return fmt.Errorf("mode is required")
|
||||
case accessMode == "":
|
||||
@@ -47,8 +56,8 @@ func (r *ImportRunsRepo) Create(ctx context.Context, run ImportRun) error {
|
||||
return fmt.Errorf("state is required")
|
||||
}
|
||||
|
||||
if _, err := r.db.ExecContext(ctx, `INSERT INTO import_runs (run_id, mode, access_mode, state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
runID, mode, accessMode, state, run.TotalItems, run.CompletedItems, run.ActiveItems, run.DegradedItems, run.BrokenItems, run.WarningItems); err != nil {
|
||||
if _, err := r.db.ExecContext(ctx, `INSERT INTO import_runs (run_id, host_id, mode, access_mode, subscription_users_json, subscription_days, probe_api_key, state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
runID, hostID, mode, accessMode, subscriptionUsersJSON, run.SubscriptionDays, probeAPIKey, state, run.TotalItems, run.CompletedItems, run.ActiveItems, run.DegradedItems, run.BrokenItems, run.WarningItems); err != nil {
|
||||
return fmt.Errorf("insert import run %q: %w", runID, err)
|
||||
}
|
||||
return nil
|
||||
@@ -56,14 +65,19 @@ func (r *ImportRunsRepo) Create(ctx context.Context, run ImportRun) error {
|
||||
|
||||
func (r *ImportRunsRepo) Update(ctx context.Context, run ImportRun) error {
|
||||
runID := strings.TrimSpace(run.RunID)
|
||||
hostID := strings.TrimSpace(run.HostID)
|
||||
mode := strings.TrimSpace(run.Mode)
|
||||
accessMode := strings.TrimSpace(run.AccessMode)
|
||||
subscriptionUsersJSON := defaultJSON(run.SubscriptionUsersJSON, "[]")
|
||||
probeAPIKey := strings.TrimSpace(run.ProbeAPIKey)
|
||||
state := strings.TrimSpace(run.State)
|
||||
finishedAt := strings.TrimSpace(run.FinishedAt)
|
||||
|
||||
switch {
|
||||
case runID == "":
|
||||
return fmt.Errorf("run_id is required")
|
||||
case hostID == "":
|
||||
return fmt.Errorf("host_id is required")
|
||||
case mode == "":
|
||||
return fmt.Errorf("mode is required")
|
||||
case accessMode == "":
|
||||
@@ -73,9 +87,9 @@ func (r *ImportRunsRepo) Update(ctx context.Context, run ImportRun) error {
|
||||
}
|
||||
|
||||
if _, err := r.db.ExecContext(ctx, `UPDATE import_runs
|
||||
SET mode = ?, access_mode = ?, state = ?, total_items = ?, completed_items = ?, active_items = ?, degraded_items = ?, broken_items = ?, warning_items = ?, finished_at = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE run_id = ?`,
|
||||
mode, accessMode, state, run.TotalItems, run.CompletedItems, run.ActiveItems, run.DegradedItems, run.BrokenItems, run.WarningItems, nullableString(finishedAt), runID); err != nil {
|
||||
SET host_id = ?, mode = ?, access_mode = ?, subscription_users_json = ?, subscription_days = ?, probe_api_key = ?, state = ?, total_items = ?, completed_items = ?, active_items = ?, degraded_items = ?, broken_items = ?, warning_items = ?, finished_at = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE run_id = ?`,
|
||||
hostID, mode, accessMode, subscriptionUsersJSON, run.SubscriptionDays, probeAPIKey, state, run.TotalItems, run.CompletedItems, run.ActiveItems, run.DegradedItems, run.BrokenItems, run.WarningItems, nullableString(finishedAt), runID); err != nil {
|
||||
return fmt.Errorf("update import run %q: %w", runID, err)
|
||||
}
|
||||
return nil
|
||||
@@ -88,8 +102,8 @@ func (r *ImportRunsRepo) GetByRunID(ctx context.Context, runID string) (ImportRu
|
||||
}
|
||||
|
||||
var run ImportRun
|
||||
if err := r.db.QueryRowContext(ctx, `SELECT run_id, mode, access_mode, state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items, started_at, updated_at, COALESCE(finished_at, '') FROM import_runs WHERE run_id = ?`, runID).
|
||||
Scan(&run.RunID, &run.Mode, &run.AccessMode, &run.State, &run.TotalItems, &run.CompletedItems, &run.ActiveItems, &run.DegradedItems, &run.BrokenItems, &run.WarningItems, &run.StartedAt, &run.UpdatedAt, &run.FinishedAt); err != nil {
|
||||
if err := r.db.QueryRowContext(ctx, `SELECT run_id, host_id, mode, access_mode, subscription_users_json, subscription_days, COALESCE(probe_api_key, ''), state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items, started_at, updated_at, COALESCE(finished_at, '') FROM import_runs WHERE run_id = ?`, runID).
|
||||
Scan(&run.RunID, &run.HostID, &run.Mode, &run.AccessMode, &run.SubscriptionUsersJSON, &run.SubscriptionDays, &run.ProbeAPIKey, &run.State, &run.TotalItems, &run.CompletedItems, &run.ActiveItems, &run.DegradedItems, &run.BrokenItems, &run.WarningItems, &run.StartedAt, &run.UpdatedAt, &run.FinishedAt); err != nil {
|
||||
return ImportRun{}, err
|
||||
}
|
||||
return run, nil
|
||||
@@ -100,7 +114,7 @@ func (r *ImportRunsRepo) List(ctx context.Context, limit int) ([]ImportRun, erro
|
||||
limit = 50
|
||||
}
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, `SELECT run_id, mode, access_mode, state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items, started_at, updated_at, COALESCE(finished_at, '') FROM import_runs ORDER BY started_at DESC LIMIT ?`, limit)
|
||||
rows, err := r.db.QueryContext(ctx, `SELECT run_id, host_id, mode, access_mode, subscription_users_json, subscription_days, COALESCE(probe_api_key, ''), state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items, started_at, updated_at, COALESCE(finished_at, '') FROM import_runs ORDER BY started_at DESC LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list import runs: %w", err)
|
||||
}
|
||||
@@ -109,7 +123,7 @@ func (r *ImportRunsRepo) List(ctx context.Context, limit int) ([]ImportRun, erro
|
||||
runs := make([]ImportRun, 0)
|
||||
for rows.Next() {
|
||||
var run ImportRun
|
||||
if err := rows.Scan(&run.RunID, &run.Mode, &run.AccessMode, &run.State, &run.TotalItems, &run.CompletedItems, &run.ActiveItems, &run.DegradedItems, &run.BrokenItems, &run.WarningItems, &run.StartedAt, &run.UpdatedAt, &run.FinishedAt); err != nil {
|
||||
if err := rows.Scan(&run.RunID, &run.HostID, &run.Mode, &run.AccessMode, &run.SubscriptionUsersJSON, &run.SubscriptionDays, &run.ProbeAPIKey, &run.State, &run.TotalItems, &run.CompletedItems, &run.ActiveItems, &run.DegradedItems, &run.BrokenItems, &run.WarningItems, &run.StartedAt, &run.UpdatedAt, &run.FinishedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan import run: %w", err)
|
||||
}
|
||||
runs = append(runs, run)
|
||||
|
||||
@@ -13,11 +13,15 @@ func TestRunStateStore(t *testing.T) {
|
||||
store := openTestDB(t)
|
||||
|
||||
run := ImportRun{
|
||||
RunID: "run-1",
|
||||
Mode: "strict",
|
||||
AccessMode: "subscription",
|
||||
State: "running",
|
||||
TotalItems: 1,
|
||||
RunID: "run-1",
|
||||
HostID: "host-1",
|
||||
Mode: "strict",
|
||||
AccessMode: "subscription",
|
||||
SubscriptionUsersJSON: `["user-1"]`,
|
||||
SubscriptionDays: 30,
|
||||
ProbeAPIKey: "probe-key",
|
||||
State: "running",
|
||||
TotalItems: 1,
|
||||
}
|
||||
if err := store.ImportRuns().Create(ctx, run); err != nil {
|
||||
t.Fatalf("ImportRuns().Create() error = %v", err)
|
||||
@@ -42,6 +46,18 @@ func TestRunStateStore(t *testing.T) {
|
||||
if gotRun.WarningItems != 1 {
|
||||
t.Fatalf("run.WarningItems = %d, want 1", gotRun.WarningItems)
|
||||
}
|
||||
if gotRun.HostID != "host-1" {
|
||||
t.Fatalf("run.HostID = %q, want host-1", gotRun.HostID)
|
||||
}
|
||||
if gotRun.SubscriptionUsersJSON != `["user-1"]` {
|
||||
t.Fatalf("run.SubscriptionUsersJSON = %q, want persisted subscription users", gotRun.SubscriptionUsersJSON)
|
||||
}
|
||||
if gotRun.SubscriptionDays != 30 {
|
||||
t.Fatalf("run.SubscriptionDays = %d, want 30", gotRun.SubscriptionDays)
|
||||
}
|
||||
if gotRun.ProbeAPIKey != "probe-key" {
|
||||
t.Fatalf("run.ProbeAPIKey = %q, want probe-key", gotRun.ProbeAPIKey)
|
||||
}
|
||||
|
||||
legacyBatchID := int64(88)
|
||||
reusedAccountID := int64(321)
|
||||
|
||||
@@ -13,6 +13,7 @@ type ReconcileRun struct {
|
||||
ProviderID int64
|
||||
Status string
|
||||
SummaryJSON string
|
||||
CreatedAt string
|
||||
}
|
||||
|
||||
type ReconcileRunsRepo struct {
|
||||
@@ -58,7 +59,7 @@ func (r *ReconcileRunsRepo) GetByBatchID(ctx context.Context, batchID int64) ([]
|
||||
return nil, fmt.Errorf("batch_id is required")
|
||||
}
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, `SELECT id, batch_id, host_id, provider_id, status, summary_json FROM reconcile_runs WHERE batch_id = ? ORDER BY id DESC`, batchID)
|
||||
rows, err := r.db.QueryContext(ctx, `SELECT id, batch_id, host_id, provider_id, status, summary_json, created_at FROM reconcile_runs WHERE batch_id = ? ORDER BY id DESC`, batchID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query reconcile runs by batch_id: %w", err)
|
||||
}
|
||||
@@ -67,7 +68,7 @@ func (r *ReconcileRunsRepo) GetByBatchID(ctx context.Context, batchID int64) ([]
|
||||
runs := make([]ReconcileRun, 0)
|
||||
for rows.Next() {
|
||||
var run ReconcileRun
|
||||
if err := rows.Scan(&run.ID, &run.BatchID, &run.HostID, &run.ProviderID, &run.Status, &run.SummaryJSON); err != nil {
|
||||
if err := rows.Scan(&run.ID, &run.BatchID, &run.HostID, &run.ProviderID, &run.Status, &run.SummaryJSON, &run.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan reconcile run by batch_id: %w", err)
|
||||
}
|
||||
runs = append(runs, run)
|
||||
@@ -86,7 +87,7 @@ func (r *ReconcileRunsRepo) GetByProviderIDAndHostID(ctx context.Context, provid
|
||||
return nil, fmt.Errorf("host_id is required")
|
||||
}
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, `SELECT id, batch_id, host_id, provider_id, status, summary_json FROM reconcile_runs WHERE provider_id = ? AND host_id = ? ORDER BY id DESC`, providerID, hostID)
|
||||
rows, err := r.db.QueryContext(ctx, `SELECT id, batch_id, host_id, provider_id, status, summary_json, created_at FROM reconcile_runs WHERE provider_id = ? AND host_id = ? ORDER BY id DESC`, providerID, hostID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query reconcile runs: %w", err)
|
||||
}
|
||||
@@ -95,7 +96,7 @@ func (r *ReconcileRunsRepo) GetByProviderIDAndHostID(ctx context.Context, provid
|
||||
runs := make([]ReconcileRun, 0)
|
||||
for rows.Next() {
|
||||
var run ReconcileRun
|
||||
if err := rows.Scan(&run.ID, &run.BatchID, &run.HostID, &run.ProviderID, &run.Status, &run.SummaryJSON); err != nil {
|
||||
if err := rows.Scan(&run.ID, &run.BatchID, &run.HostID, &run.ProviderID, &run.Status, &run.SummaryJSON, &run.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan reconcile run: %w", err)
|
||||
}
|
||||
runs = append(runs, run)
|
||||
|
||||
56
internal/store/sqlite/reconcile_runs_repo_test.go
Normal file
56
internal/store/sqlite/reconcile_runs_repo_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReconcileRunsRepoPersistsCreatedAt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := openTestDB(t)
|
||||
hostID := createTestHostWithBaseURL(t, store, "host-1", "https://host.example")
|
||||
packID := createTestPack(t, store)
|
||||
providerID, err := store.Providers().Create(context.Background(), Provider{
|
||||
PackID: packID,
|
||||
ProviderID: "provider-1",
|
||||
DisplayName: "Provider 1",
|
||||
BaseURL: "https://provider.example",
|
||||
Platform: "openai",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Providers().Create() error = %v", err)
|
||||
}
|
||||
batchID, err := store.ImportBatches().Create(context.Background(), ImportBatch{
|
||||
HostID: hostID,
|
||||
PackID: packID,
|
||||
ProviderID: providerID,
|
||||
Mode: "partial",
|
||||
BatchStatus: "succeeded",
|
||||
AccessStatus: "self_service_ready",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ImportBatches().Create() error = %v", err)
|
||||
}
|
||||
|
||||
if _, err := store.ReconcileRuns().Create(context.Background(), ReconcileRun{
|
||||
BatchID: batchID,
|
||||
HostID: hostID,
|
||||
ProviderID: providerID,
|
||||
Status: "active",
|
||||
SummaryJSON: `{"ok":true}`,
|
||||
}); err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
|
||||
runs, err := store.ReconcileRuns().GetByProviderIDAndHostID(context.Background(), providerID, hostID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetByProviderIDAndHostID() error = %v", err)
|
||||
}
|
||||
if len(runs) != 1 {
|
||||
t.Fatalf("reconcile runs = %d, want 1", len(runs))
|
||||
}
|
||||
if runs[0].CreatedAt == "" {
|
||||
t.Fatal("CreatedAt = empty, want timestamp")
|
||||
}
|
||||
}
|
||||
65
internal/worker/runner.go
Normal file
65
internal/worker/runner.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Job interface {
|
||||
Name() string
|
||||
Run(context.Context) error
|
||||
}
|
||||
|
||||
type Logger func(format string, args ...any)
|
||||
|
||||
type Runner struct {
|
||||
jobs []Job
|
||||
interval time.Duration
|
||||
logger Logger
|
||||
}
|
||||
|
||||
func NewRunner(jobs []Job, interval time.Duration, logger Logger) *Runner {
|
||||
return &Runner{
|
||||
jobs: append([]Job(nil), jobs...),
|
||||
interval: interval,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) Start(ctx context.Context) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
r.runOnce(ctx)
|
||||
if ctx.Err() != nil || r.interval <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(r.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
r.runOnce(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (r *Runner) runOnce(ctx context.Context) {
|
||||
for _, job := range r.jobs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if job == nil {
|
||||
continue
|
||||
}
|
||||
if err := job.Run(ctx); err != nil && ctx.Err() == nil && r.logger != nil {
|
||||
r.logger("%s: %v", job.Name(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
53
internal/worker/runner_test.go
Normal file
53
internal/worker/runner_test.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRunnerStartRunsJobsImmediatelyAndOnSchedule(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
job := &stubJob{}
|
||||
runner := NewRunner([]Job{job}, 10*time.Millisecond, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
runner.Start(ctx)
|
||||
|
||||
deadline := time.Now().Add(250 * time.Millisecond)
|
||||
for {
|
||||
if job.Count() >= 2 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("job count = %d, want at least 2 scheduled runs", job.Count())
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
type stubJob struct {
|
||||
mu sync.Mutex
|
||||
count int
|
||||
}
|
||||
|
||||
func (j *stubJob) Name() string {
|
||||
return "stub"
|
||||
}
|
||||
|
||||
func (j *stubJob) Run(context.Context) error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
j.count++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *stubJob) Count() int {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.count
|
||||
}
|
||||
@@ -139,6 +139,17 @@ func TestStoreAppliesLatestMigration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
for _, column := range []string{
|
||||
"host_id",
|
||||
"subscription_users_json",
|
||||
"subscription_days",
|
||||
"probe_api_key",
|
||||
} {
|
||||
if !tableColumnExists(t, store.SQLDB(), "import_runs", column) {
|
||||
t.Fatalf("column %q missing from import_runs", column)
|
||||
}
|
||||
}
|
||||
|
||||
for _, column := range []string{
|
||||
"api_key_fingerprint",
|
||||
"canonical_model_families_json",
|
||||
|
||||
Reference in New Issue
Block a user