forked from niuniu/llm-intelligence
feat(audit): add pricing signature guards and reporting
Add snapshot, signature, and drift guard support for Vertex AI, Cloudflare Workers AI, and Perplexity API, backed by a queryable audit table and recent-window view. This commit also wires the audit query layer into daily signal materialization and report generation so structure drift becomes a first-class signal instead of a log-only artifact.
This commit is contained in:
phamnazage-jpg
@@ -260,6 +260,32 @@ func TestResolveReportRunContextMarksHistoricalRebuildAsNonOfficial(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveSignatureAuditReportConfigDefaults(t *testing.T) {
|
||||
t.Setenv("REPORT_SIGNATURE_AUDIT_WINDOW", "")
|
||||
t.Setenv("REPORT_SIGNATURE_AUDIT_CHANGED_THRESHOLD", "")
|
||||
|
||||
cfg := resolveSignatureAuditReportConfig()
|
||||
if cfg.Window != 5 {
|
||||
t.Fatalf("window = %d, want 5", cfg.Window)
|
||||
}
|
||||
if cfg.ChangedRunsThreshold != 1 {
|
||||
t.Fatalf("changed threshold = %d, want 1", cfg.ChangedRunsThreshold)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveSignatureAuditReportConfigReadsEnvOverride(t *testing.T) {
|
||||
t.Setenv("REPORT_SIGNATURE_AUDIT_WINDOW", "9")
|
||||
t.Setenv("REPORT_SIGNATURE_AUDIT_CHANGED_THRESHOLD", "3")
|
||||
|
||||
cfg := resolveSignatureAuditReportConfig()
|
||||
if cfg.Window != 9 {
|
||||
t.Fatalf("window = %d, want 9", cfg.Window)
|
||||
}
|
||||
if cfg.ChangedRunsThreshold != 3 {
|
||||
t.Fatalf("changed threshold = %d, want 3", cfg.ChangedRunsThreshold)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeTrackedSummaryPrependsRuntimeAudit(t *testing.T) {
|
||||
summary := composeTrackedSummary(
|
||||
"models=42 free=3 intl=5 domestic=10",
|
||||
@@ -498,7 +524,7 @@ func TestGenerateMarkdownV3IncludesTencentSubscriptionSection(t *testing.T) {
|
||||
"主来源: OpenRouter / region_pricing",
|
||||
"更新时间: 2026-05-13 09:30",
|
||||
"判定依据: models.created_at = 今日，且已存在最新价格快照",
|
||||
"## 💳 腾讯云套餐订阅价",
|
||||
"## 💳 中转平台套餐订阅价",
|
||||
"通用 Token Plan Lite",
|
||||
"Hy Token Plan Max",
|
||||
"¥39.00/月",
|
||||
@@ -618,7 +644,185 @@ func TestGenerateHTMLV3IncludesTencentSubscriptionSection(t *testing.T) {
|
||||
"官方免费",
|
||||
"聚合免费",
|
||||
"待确认",
|
||||
"💳 腾讯云套餐订阅价",
|
||||
"💳 中转平台套餐订阅价",
|
||||
} {
|
||||
if !strings.Contains(content, want) {
|
||||
t.Fatalf("html missing %q\n%s", want, content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateHTMLV3IncludesResellerSubscriptionComparison(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "daily_report.html")
|
||||
report := sampleReportForV1()
|
||||
report.TencentSubscriptionPlans = []SubscriptionPlanInfo{
|
||||
{
|
||||
ProviderName: "Tencent",
|
||||
OperatorName: "Tencent Cloud",
|
||||
PlanName: "通用 Token Plan 首月活动版",
|
||||
PlanFamily: "token_plan",
|
||||
Tier: "首月活动版",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: 19,
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: 35000000,
|
||||
QuotaUnit: "tokens/month",
|
||||
ContextWindow: 131072,
|
||||
ModelCount: 2,
|
||||
ModelPreview: "glm-5, hunyuan-t1",
|
||||
Notes: "首购用户首月优惠，次月恢复标准价。",
|
||||
EffectiveDate: "2026-05-14",
|
||||
},
|
||||
{
|
||||
ProviderName: "Alibaba",
|
||||
OperatorName: "Alibaba Cloud Bailian",
|
||||
PlanName: "百炼 Coding Plan 首购版",
|
||||
PlanFamily: "coding_plan",
|
||||
Tier: "首购版",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: 29,
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: 50000000,
|
||||
QuotaUnit: "tokens/month",
|
||||
ContextWindow: 262144,
|
||||
ModelCount: 3,
|
||||
ModelPreview: "qwen-coder-plus, qwen3-coder, deepseek-r1",
|
||||
Notes: "首月活动价，适合低成本试用编码模型。",
|
||||
EffectiveDate: "2026-05-14",
|
||||
},
|
||||
}
|
||||
decorateReportV1(report)
|
||||
|
||||
if err := generateHTMLV3(report, path); err != nil {
|
||||
t.Fatalf("generateHTMLV3 returned error: %v", err)
|
||||
}
|
||||
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read html output: %v", err)
|
||||
}
|
||||
|
||||
content := string(body)
|
||||
for _, want := range []string{
|
||||
"💳 中转平台套餐订阅价",
|
||||
"Tencent Cloud",
|
||||
"Alibaba Cloud Bailian",
|
||||
"通用 Token Plan 首月活动版",
|
||||
"百炼 Coding Plan 首购版",
|
||||
"首购用户首月优惠，次月恢复标准价。",
|
||||
"首月活动价，适合低成本试用编码模型。",
|
||||
"Token Plan",
|
||||
"Coding Plan",
|
||||
} {
|
||||
if !strings.Contains(content, want) {
|
||||
t.Fatalf("html missing %q\n%s", want, content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMarkdownV3IncludesSignatureStabilitySection(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "daily_report.md")
|
||||
report := sampleReportForV1()
|
||||
report.SignatureAuditSummaries = []SignatureAuditSourceSummary{
|
||||
{
|
||||
SourceKey: "cloudflare_pricing_signature",
|
||||
SourceLabel: "Cloudflare Workers AI",
|
||||
RunsInWindow: 5,
|
||||
ChangedRuns: 2,
|
||||
LatestStatus: "passed",
|
||||
LatestStructureState: "stable",
|
||||
LatestCheckedAt: "2026-05-15 20:01:46",
|
||||
},
|
||||
{
|
||||
SourceKey: "vertex_pricing_signature",
|
||||
SourceLabel: "Google Cloud Vertex AI",
|
||||
RunsInWindow: 5,
|
||||
ChangedRuns: 0,
|
||||
LatestStatus: "passed",
|
||||
LatestStructureState: "stable",
|
||||
LatestCheckedAt: "2026-05-15 19:47:11",
|
||||
},
|
||||
}
|
||||
report.SignatureAuditRows = []SignatureAuditReportRow{
|
||||
{
|
||||
SourceKey: "cloudflare_pricing_signature",
|
||||
SourceLabel: "Cloudflare Workers AI",
|
||||
RecentRank: 2,
|
||||
CheckedAt: "2026-05-14 20:01:46",
|
||||
StructureState: "changed",
|
||||
StructureChanged: true,
|
||||
Status: "drift_detected",
|
||||
StructureSHA256: "def456",
|
||||
},
|
||||
}
|
||||
decorateReportV1(report)
|
||||
|
||||
if err := generateMarkdownV3(report, path); err != nil {
|
||||
t.Fatalf("generateMarkdownV3 returned error: %v", err)
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read markdown output: %v", err)
|
||||
}
|
||||
content := string(body)
|
||||
for _, want := range []string{
|
||||
"## 结构稳定性",
|
||||
"Cloudflare Workers AI",
|
||||
"Google Cloud Vertex AI",
|
||||
"最近 5 次中出现 2 次结构变化",
|
||||
"changed",
|
||||
"def456",
|
||||
} {
|
||||
if !strings.Contains(content, want) {
|
||||
t.Fatalf("markdown missing %q\n%s", want, content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateHTMLV3IncludesSignatureStabilitySection(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "daily_report.html")
|
||||
report := sampleReportForV1()
|
||||
report.SignatureAuditSummaries = []SignatureAuditSourceSummary{
|
||||
{
|
||||
SourceKey: "perplexity_pricing_signature",
|
||||
SourceLabel: "Perplexity API",
|
||||
RunsInWindow: 5,
|
||||
ChangedRuns: 1,
|
||||
LatestStatus: "baseline_initialized",
|
||||
LatestStructureState: "initial",
|
||||
LatestCheckedAt: "2026-05-15 20:01:46",
|
||||
},
|
||||
}
|
||||
report.SignatureAuditRows = []SignatureAuditReportRow{
|
||||
{
|
||||
SourceKey: "perplexity_pricing_signature",
|
||||
SourceLabel: "Perplexity API",
|
||||
RecentRank: 1,
|
||||
CheckedAt: "2026-05-15 20:01:46",
|
||||
StructureState: "initial",
|
||||
StructureChanged: false,
|
||||
Status: "baseline_initialized",
|
||||
StructureSHA256: "abc123",
|
||||
},
|
||||
}
|
||||
decorateReportV1(report)
|
||||
|
||||
if err := generateHTMLV3(report, path); err != nil {
|
||||
t.Fatalf("generateHTMLV3 returned error: %v", err)
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read html output: %v", err)
|
||||
}
|
||||
content := string(body)
|
||||
for _, want := range []string{
|
||||
"结构稳定性",
|
||||
"Perplexity API",
|
||||
"最近 5 次中出现 1 次结构变化",
|
||||
"baseline_initialized",
|
||||
"abc123",
|
||||
} {
|
||||
if !strings.Contains(content, want) {
|
||||
t.Fatalf("html missing %q\n%s", want, content)
|
||||
@@ -746,6 +950,162 @@ func TestBuildHeadlineItemsDeduplicatesSameModel(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHeadlineItemsElevatesSignatureDrift(t *testing.T) {
|
||||
report := sampleReportForV1()
|
||||
report.SignatureAuditConfig = SignatureAuditReportConfig{Window: 5, ChangedRunsThreshold: 2}
|
||||
report.ModelEvents = []ModelEvent{
|
||||
{
|
||||
EventType: "new_model",
|
||||
ModelName: "DeepSeek-V4-Flash",
|
||||
ProviderName: "DeepSeek",
|
||||
TrustLabel: "聚合来源",
|
||||
Baseline: "首次出现",
|
||||
Summary: "新模型进入情报池。",
|
||||
SourceKindLabel: "模型快照",
|
||||
PrimarySource: "OpenRouter / region_pricing",
|
||||
UpdatedAt: "2026-05-13 09:30",
|
||||
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
|
||||
Priority: 90,
|
||||
},
|
||||
}
|
||||
report.SignatureAuditSummaries = []SignatureAuditSourceSummary{
|
||||
{
|
||||
SourceKey: "cloudflare_pricing_signature",
|
||||
SourceLabel: "Cloudflare Workers AI",
|
||||
RunsInWindow: 5,
|
||||
ChangedRuns: 2,
|
||||
LatestCheckedAt: "2026-05-15 20:01:46",
|
||||
LatestStatus: "drift_detected",
|
||||
LatestStructureState: "changed",
|
||||
},
|
||||
}
|
||||
|
||||
items := buildHeadlineItems(report)
|
||||
if len(items) == 0 {
|
||||
t.Fatalf("expected headline items")
|
||||
}
|
||||
if items[0].Label != "结构波动" {
|
||||
t.Fatalf("expected signature drift headline first, got %+v", items[0])
|
||||
}
|
||||
if !strings.Contains(items[0].Title, "Cloudflare Workers AI") {
|
||||
t.Fatalf("expected drift headline title to mention source, got %+v", items[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildActionItemsElevatesSignatureDrift(t *testing.T) {
|
||||
report := sampleReportForV1()
|
||||
report.SignatureAuditConfig = SignatureAuditReportConfig{Window: 5, ChangedRunsThreshold: 1}
|
||||
report.SignatureAuditSummaries = []SignatureAuditSourceSummary{
|
||||
{
|
||||
SourceKey: "perplexity_pricing_signature",
|
||||
SourceLabel: "Perplexity API",
|
||||
RunsInWindow: 5,
|
||||
ChangedRuns: 1,
|
||||
LatestCheckedAt: "2026-05-15 20:01:46",
|
||||
LatestStatus: "drift_detected",
|
||||
LatestStructureState: "changed",
|
||||
},
|
||||
}
|
||||
decorateReportV1(report)
|
||||
|
||||
if len(report.ActionItems) == 0 {
|
||||
t.Fatalf("expected action items")
|
||||
}
|
||||
if !strings.Contains(report.ActionItems[0].Title, "Perplexity API") {
|
||||
t.Fatalf("expected first action item to elevate signature drift, got %+v", report.ActionItems[0])
|
||||
}
|
||||
if !strings.Contains(report.ActionItems[0].Evidence, "最近 5 次中出现 1 次结构变化") {
|
||||
t.Fatalf("expected signature drift evidence, got %+v", report.ActionItems[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHeadlineItemsDoesNotElevateSignatureDriftBelowThreshold(t *testing.T) {
|
||||
report := sampleReportForV1()
|
||||
report.SignatureAuditConfig = SignatureAuditReportConfig{Window: 5, ChangedRunsThreshold: 3}
|
||||
report.ModelEvents = []ModelEvent{
|
||||
{
|
||||
EventType: "new_model",
|
||||
ModelName: "DeepSeek-V4-Flash",
|
||||
ProviderName: "DeepSeek",
|
||||
TrustLabel: "聚合来源",
|
||||
Baseline: "首次出现",
|
||||
Summary: "新模型进入情报池。",
|
||||
SourceKindLabel: "模型快照",
|
||||
PrimarySource: "OpenRouter / region_pricing",
|
||||
UpdatedAt: "2026-05-13 09:30",
|
||||
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
|
||||
Priority: 90,
|
||||
},
|
||||
}
|
||||
report.SignatureAuditSummaries = []SignatureAuditSourceSummary{
|
||||
{
|
||||
SourceKey: "cloudflare_pricing_signature",
|
||||
SourceLabel: "Cloudflare Workers AI",
|
||||
RunsInWindow: 5,
|
||||
ChangedRuns: 2,
|
||||
LatestCheckedAt: "2026-05-15 20:01:46",
|
||||
LatestStatus: "drift_detected",
|
||||
LatestStructureState: "changed",
|
||||
},
|
||||
}
|
||||
|
||||
items := buildHeadlineItems(report)
|
||||
if len(items) == 0 {
|
||||
t.Fatalf("expected headline items")
|
||||
}
|
||||
if items[0].Label == "结构波动" {
|
||||
t.Fatalf("signature drift should stay below threshold, got %+v", items[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecorateReportV1ElevatesSignatureDriftIntoHeroSummary(t *testing.T) {
|
||||
report := sampleReportForV1()
|
||||
report.ModelEvents = nil
|
||||
report.SignatureAuditConfig = SignatureAuditReportConfig{Window: 5, ChangedRunsThreshold: 2}
|
||||
report.SignatureAuditSummaries = []SignatureAuditSourceSummary{
|
||||
{
|
||||
SourceKey: "vertex_pricing_signature",
|
||||
SourceLabel: "Google Cloud Vertex AI",
|
||||
RunsInWindow: 5,
|
||||
ChangedRuns: 3,
|
||||
LatestCheckedAt: "2026-05-15 20:01:46",
|
||||
LatestStatus: "drift_detected",
|
||||
LatestStructureState: "changed",
|
||||
},
|
||||
}
|
||||
|
||||
decorateReportV1(report)
|
||||
|
||||
if !strings.Contains(report.HeroSummary, "Google Cloud Vertex AI") {
|
||||
t.Fatalf("expected hero summary to mention signature drift source, got %q", report.HeroSummary)
|
||||
}
|
||||
if !strings.Contains(report.HeroEvidence, "最近 5 次中出现 3 次结构变化") {
|
||||
t.Fatalf("expected hero evidence to mention drift count, got %q", report.HeroEvidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignatureAuditSummaryToneRespectsConfiguredThreshold(t *testing.T) {
|
||||
report := &ReportV3{
|
||||
SignatureAuditConfig: SignatureAuditReportConfig{Window: 5, ChangedRunsThreshold: 3},
|
||||
}
|
||||
|
||||
if tone := signatureAuditSummaryTone(report, SignatureAuditSourceSummary{
|
||||
SourceLabel: "Cloudflare Workers AI",
|
||||
ChangedRuns: 2,
|
||||
RunsInWindow: 5,
|
||||
}); tone != "official" {
|
||||
t.Fatalf("tone below threshold = %q, want official", tone)
|
||||
}
|
||||
|
||||
if tone := signatureAuditSummaryTone(report, SignatureAuditSourceSummary{
|
||||
SourceLabel: "Cloudflare Workers AI",
|
||||
ChangedRuns: 3,
|
||||
RunsInWindow: 5,
|
||||
}); tone != "warning" {
|
||||
t.Fatalf("tone at threshold = %q, want warning", tone)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadlineItemFromModelEventIncludesEvidenceFields(t *testing.T) {
|
||||
item := headlineItemFromModelEvent(ModelEvent{
|
||||
EventType: "new_model",
|
||||
|
||||
Reference in New Issue
Block a user