feat(imports): add real pricing and subscription collectors

Add plan catalog and subscription schema support, seed baselines, and real importers for core domestic subscriptions plus stable official pricing sources. This commit also hardens the shared fetch layers so the importers can support live collection and database writes instead of relying on manual placeholders alone.
2026-05-15 22:32:57 +08:00
parent dd58c18fe3
commit 958245537a
91 changed files with 10474 additions and 1 deletions
--- a/scripts/import_bedrock_pricing_test.go
+++ b/scripts/import_bedrock_pricing_test.go
@@ -0,0 +1,130 @@
+//go:build llm_script
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestParseBedrockPricingCatalogBuildsRecords(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("testdata", "bedrock_pricing_sample.html"))
+	if err != nil {
+		t.Fatalf("读取 fixture 失败: %v", err)
+	}
+
+	records, err := parseBedrockPricingCatalog(string(raw))
+	if err != nil {
+		t.Fatalf("parseBedrockPricingCatalog 返回错误: %v", err)
+	}
+	if len(records) != 4 {
+		t.Fatalf("期望 4 条 Bedrock 价格记录，实际 %d", len(records))
+	}
+	if records[0].ProviderName != "Amazon" {
+		t.Fatalf("Nova provider 归一化错误: %q", records[0].ProviderName)
+	}
+	if records[1].OutputPrice != 0.4 {
+		t.Fatalf("Nova Lite 输出价错误: %v", records[1].OutputPrice)
+	}
+	if records[2].Region != "Europe (Frankfurt) and Asia Pacific (Jakarta)" {
+		t.Fatalf("Qwen region 错误: %q", records[2].Region)
+	}
+}
+
+func TestRunBedrockPricingImportDryRunPrintsSummary(t *testing.T) {
+	var out bytes.Buffer
+	err := runBedrockPricingImport(bedrockPricingImportConfig{
+		URL:     defaultBedrockPricingURL,
+		Fixture: filepath.Join("testdata", "bedrock_pricing_sample.html"),
+		DryRun:  true,
+	}, nil, &out)
+	if err != nil {
+		t.Fatalf("runBedrockPricingImport 返回错误: %v", err)
+	}
+	output := out.String()
+	for _, want := range []string{
+		"source=bedrock-pricing-import",
+		"models=4",
+		"operator=Amazon Bedrock",
+		"dry_run=true",
+	} {
+		if !strings.Contains(output, want) {
+			t.Fatalf("输出缺少 %q，实际: %q", want, output)
+		}
+	}
+}
+
+func TestParseBedrockPricingCatalogPrefersFirstTierForSameModelAndRegion(t *testing.T) {
+	raw := `
+<h3 id="Model_Pricing" class="lb-txt-none lb-txt-32 lb-h3 lb-title">Model Pricing</h3>
+<h2 id="Anthropic" class="lb-txt-none lb-h2 lb-title">Anthropic</h2>
+<p><b>Regions:&nbsp;US East (N. Virginia)</b></p>
+<table>
+  <tbody>
+    <tr>
+      <td><b>Anthropic models</b></td>
+      <td><b>Price per 1M input tokens</b></td>
+      <td><b>Price per 1M output tokens</b></td>
+    </tr>
+    <tr>
+      <td>Claude Sonnet 4.5</td>
+      <td>$3.00</td>
+      <td>$15.00</td>
+    </tr>
+  </tbody>
+</table>
+<h3 id="Priority">Priority</h3>
+<table>
+  <tbody>
+    <tr>
+      <td><b>Anthropic models</b></td>
+      <td><b>Price per 1M input tokens</b></td>
+      <td><b>Price per 1M output tokens</b></td>
+    </tr>
+    <tr>
+      <td>Claude Sonnet 4.5</td>
+      <td>$6.00</td>
+      <td>$30.00</td>
+    </tr>
+  </tbody>
+</table>
+<h3 id="Pricing_examples" class="lb-txt-none lb-txt-48 lb-h2 lb-title">Pricing examples</h3>
+`
+
+	records, err := parseBedrockPricingCatalog(raw)
+	if err != nil {
+		t.Fatalf("parseBedrockPricingCatalog 返回错误: %v", err)
+	}
+	if len(records) != 1 {
+		t.Fatalf("期望仅保留默认价一条记录，实际 %d", len(records))
+	}
+	if records[0].InputPrice != 3 || records[0].OutputPrice != 15 {
+		t.Fatalf("默认价被后续价阶污染: %+v", records[0])
+	}
+}
+
+func TestParseBedrockPricingTextFallbackBuildsRecords(t *testing.T) {
+	raw := cleanHTMLText(`
+<h2 id="Qwen" class="lb-txt-none lb-h2 lb-title">Qwen</h2>
+<p><b>Regions:&nbsp;Europe (Frankfurt) and&nbsp;Asia Pacific (Jakarta)</b></p>
+Qwen models Price per 1M input tokens Price per 1M output tokens
+Qwen3 Coder Next $ 0.60 $ 1.44
+<p><b>Region: Asia Pacific (Sydney)</b></p>
+Qwen models Price per 1M input tokens Price per 1M output tokens
+Qwen3 Next 80B A3B $ 0.1545 $ 1.2360
+`)
+
+	records := parseBedrockPricingTextFallback(raw)
+	if len(records) != 2 {
+		t.Fatalf("期望 fallback 解析 2 条记录，实际 %d", len(records))
+	}
+	if records[0].Region != "Europe (Frankfurt) and Asia Pacific (Jakarta)" {
+		t.Fatalf("fallback region 错误: %q", records[0].Region)
+	}
+	if records[1].InputPrice != 0.1545 || records[1].OutputPrice != 1.2360 {
+		t.Fatalf("fallback 价格错误: %+v", records[1])
+	}
+}