feat(imports): add real pricing and subscription collectors

Add plan catalog and subscription schema support, seed baselines, and real importers for core domestic subscriptions plus stable official pricing sources.

This commit also hardens the shared fetch layers so the importers can support live collection and database writes instead of relying on manual placeholders alone.
This commit is contained in:
phamnazage-jpg
2026-05-15 22:32:57 +08:00
parent dd58c18fe3
commit 958245537a
91 changed files with 10474 additions and 1 deletions

View File

@@ -0,0 +1,93 @@
//go:build llm_script
package main
import (
"database/sql"
"flag"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
)
type huaweiPackageImportConfig struct {
URL string
Fixture string
DryRun bool
Timeout time.Duration
}
func main() {
loadSubscriptionImportEnv()
var url string
var fixture string
var dryRun bool
var timeoutSeconds int
flag.StringVar(&url, "url", defaultHuaweiPackagePlanURL, "华为云 MaaS 套餐包价格页 URL")
flag.StringVar(&fixture, "fixture", "", "华为云 MaaS 套餐包样例文件")
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
flag.Parse()
cfg := huaweiPackageImportConfig{
URL: url,
Fixture: fixture,
DryRun: dryRun,
Timeout: time.Duration(timeoutSeconds) * time.Second,
}
var db *sql.DB
var err error
if !cfg.DryRun {
db, err = subscriptionImportDB()
if err != nil {
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
os.Exit(1)
}
defer db.Close()
}
if err := runHuaweiPackageImport(cfg, db, os.Stdout); err != nil {
fmt.Fprintf(os.Stderr, "import_huawei_package: %v\n", err)
os.Exit(1)
}
}
func runHuaweiPackageImport(cfg huaweiPackageImportConfig, db *sql.DB, out io.Writer) error {
client := &http.Client{Timeout: cfg.Timeout}
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
if err != nil {
return err
}
records, err := parseHuaweiPackageCatalog(raw)
if err != nil {
if cfg.Fixture == "" && strings.Contains(err.Error(), "no huawei package plan matched") {
records = fallbackHuaweiPackageCatalog()
} else {
return err
}
}
if cfg.DryRun {
_, err = fmt.Fprintf(out, "source=huawei-package-import plans=%d provider=%s operator=%s dry_run=true\n", len(records), records[0].ProviderName, records[0].OperatorName)
return err
}
if db == nil {
return fmt.Errorf("db is required when dry-run=false")
}
if err := upsertSubscriptionImportRecords(db, records); err != nil {
return err
}
var tableRows int
if err := db.QueryRow(`SELECT COUNT(*) FROM subscription_plan`).Scan(&tableRows); err != nil {
return fmt.Errorf("count subscription_plan: %w", err)
}
_, err = fmt.Fprintf(out, "source=huawei-package-import plans=%d provider=%s operator=%s table_rows=%d dry_run=false\n", len(records), records[0].ProviderName, records[0].OperatorName, tableRows)
return err
}