xingxing/llm-intelligence
1
0
Fork 0
forked from niuniu/llm-intelligence
Code Pull Requests Activity

chore: prepare repository for publishing

Browse Source
This commit is contained in:
phamnazage-jpg
2026-05-13 14:42:45 +08:00
parent 55e506b2b5
commit 77e6610fd2
118 changed files with 27373 additions and 1009 deletions
Download Patch File Download Diff File Expand all files Collapse all files

280
scripts/verification_executor.go
View File

@@ -3,6 +3,8 @@
// matches expected_evidence, outputs pass/fail report.
//
// Usage: go run scripts/verification_executor.go [--dry-run] [--task T-Q2-1.1]
//go:build llm_script
package main
import (
@@ -21,28 +23,36 @@ import (
)
type Verification struct {
Mode string
Command string
Mode string
Command string
ExpectedEvidence string
TimeoutSeconds int
TimeoutSeconds int
EvidenceGrade string
TaskType string
}
type TaskResult struct {
TaskID string
TaskName string
Verified bool
Command string
ExitCode int
Stdout string
Stderr string
Error string
Reason string
TaskID string
TaskName string
Verified bool
Command string
ExitCode int
Stdout string
Stderr string
StdoutSummary string
StderrSummary string
Error string
Reason string
EvidenceGrade string
TaskType string
}
func main() {
dryRun := flag.Bool("dry-run", false, "print commands without executing")
taskFilter := flag.String("task", "", "filter by task ID (e.g. T-Q2-1.1)")
tasksPathFlag := flag.String("tasks", "", "path to TASKS.md")
statusFilter := flag.String("status", "all", "filter by normalized status: all|completed|in_progress|planned|paused|unknown")
completedOnly := flag.Bool("completed-only", false, "shortcut for --status completed")
flag.Parse()
tasksPath := resolveTasksPath(*tasksPathFlag)
@@ -65,8 +75,18 @@ func main() {
tasks = filtered
}
effectiveStatus := *statusFilter
if *completedOnly {
effectiveStatus = "completed"
}
tasks, err = filterTasksByStatus(tasks, effectiveStatus)
if err != nil {
fmt.Fprintf(os.Stderr, "filter tasks: %v\n", err)
os.Exit(1)
}
fmt.Printf("=== Verification Report (%s) ===\n", time.Now().Format("2006-01-02 15:04"))
fmt.Printf("Tasks checked: %d | Dry-run: %v | TASKS: %s\n\n", len(tasks), *dryRun, tasksPath)
fmt.Printf("Tasks checked: %d | Dry-run: %v | Status: %s | TASKS: %s\n\n", len(tasks), *dryRun, effectiveStatus, tasksPath)
var passed, failed int
var results []TaskResult
@@ -87,17 +107,24 @@ func main() {
icon = "❌"
}
fmt.Printf("%s [%s] %s\n", icon, r.TaskID, r.TaskName)
if r.Command != "" {
fmt.Printf(" cmd: %s\n", r.Command)
}
if r.EvidenceGrade != "" || r.TaskType != "" {
fmt.Printf(" grade: %s | type: %s\n", r.EvidenceGrade, r.TaskType)
}
if r.StderrSummary != "" {
fmt.Printf(" stderr: %s\n", r.StderrSummary)
}
if r.StdoutSummary != "" && (!r.Verified || r.Reason != "" || r.Error != "") {
fmt.Printf(" stdout: %s\n", r.StdoutSummary)
}
if r.Error != "" {
fmt.Printf(" ERROR: %s\n", r.Error)
} else {
if r.Command != "" {
fmt.Printf(" cmd: %s\n", r.Command)
}
if r.ExitCode != 0 && r.Stdout != "" {
fmt.Printf(" output: %s\n", strings.TrimSpace(r.Stdout))
} else if r.Reason != "" {
fmt.Printf(" reason: %s\n", r.Reason)
}
} else if r.ExitCode != 0 && r.Stdout != "" {
fmt.Printf(" output: %s\n", strings.TrimSpace(r.Stdout))
} else if r.Reason != "" {
fmt.Printf(" reason: %s\n", r.Reason)
}
}
@@ -108,28 +135,40 @@ func main() {
}
func resolveTasksPath(flagValue string) string {
envValue := os.Getenv("TASKS_PATH")
wd := ""
if currentWD, err := os.Getwd(); err == nil {
wd = currentWD
}
sourceDir := ""
if _, sourcePath, _, ok := runtime.Caller(0); ok {
sourceDir = filepath.Dir(sourcePath)
}
return resolveTasksPathWithContext(flagValue, envValue, wd, sourceDir, "/home/long/.openclaw/workspace/TASKS.md")
}
func resolveTasksPathWithContext(flagValue, envValue, wd, sourceDir, globalTasksPath string) string {
candidates := []string{}
if flagValue != "" {
candidates = append(candidates, flagValue)
}
if envValue := os.Getenv("TASKS_PATH"); envValue != "" {
if envValue != "" {
candidates = append(candidates, envValue)
}
if wd, err := os.Getwd(); err == nil {
if wd != "" {
candidates = append(candidates,
filepath.Join(wd, "TASKS.md"),
filepath.Join(wd, "..", "TASKS.md"),
)
}
if _, sourcePath, _, ok := runtime.Caller(0); ok {
scriptDir := filepath.Dir(sourcePath)
candidates = append(candidates, filepath.Join(scriptDir, "..", "TASKS.md"))
defaultProjectTasks := ""
if sourceDir != "" {
defaultProjectTasks = filepath.Join(sourceDir, "..", "TASKS.md")
candidates = append(candidates, defaultProjectTasks)
}
candidates = append(candidates, "/home/long/.openclaw/workspace/TASKS.md")
seen := map[string]struct{}{}
for _, candidate := range candidates {
if candidate == "" {
@@ -148,16 +187,26 @@ func resolveTasksPath(flagValue string) string {
if flagValue != "" {
return filepath.Clean(flagValue)
}
if envValue := os.Getenv("TASKS_PATH"); envValue != "" {
if envValue != "" {
return filepath.Clean(envValue)
}
return "/home/long/.openclaw/workspace/TASKS.md"
if defaultProjectTasks != "" {
return filepath.Clean(defaultProjectTasks)
}
if wd != "" {
return filepath.Clean(filepath.Join(wd, "TASKS.md"))
}
if globalTasksPath != "" {
return filepath.Clean(globalTasksPath)
}
return "TASKS.md"
}
type taskEntry struct {
ID string
Name string
Verification Verification
ID string
Name string
Status string
Verification Verification
HasVerification bool
}
@@ -176,7 +225,7 @@ func parseTasks(f *os.File) []taskEntry {
if currentTask != nil {
tasks = append(tasks, *currentTask)
}
currentTask = &taskEntry{ID: m[1], Name: m[2]}
currentTask = &taskEntry{ID: m[1], Name: m[2], Status: normalizeStatusFromText(line)}
inVerification = false
continue
}
@@ -193,6 +242,10 @@ func parseTasks(f *os.File) []taskEntry {
}
if !inVerification {
statusRe := regexp.MustCompile(`^\s*-\s+\*\*状态\*\*：(.+)$`)
if m := statusRe.FindStringSubmatch(line); m != nil {
currentTask.Status = normalizeStatusFromText(m[1])
}
continue
}
@@ -216,6 +269,18 @@ func parseTasks(f *os.File) []taskEntry {
continue
}
evidenceGradeRe := regexp.MustCompile(`^\s+- evidence_grade:\s+` + "`" + `([^` + "`" + `]+)` + "`")
if m := evidenceGradeRe.FindStringSubmatch(line); m != nil {
currentTask.Verification.EvidenceGrade = m[1]
continue
}
taskTypeRe := regexp.MustCompile(`^\s+- task_type:\s+` + "`" + `([^` + "`" + `]+)` + "`")
if m := taskTypeRe.FindStringSubmatch(line); m != nil {
currentTask.Verification.TaskType = m[1]
continue
}
timeoutRe := regexp.MustCompile(`^\s+- timeout_seconds:\s+(\d+)`)
if m := timeoutRe.FindStringSubmatch(line); m != nil {
fmt.Sscanf(m[1], "%d", &currentTask.Verification.TimeoutSeconds)
@@ -244,6 +309,18 @@ func verifyTask(t taskEntry, dryRun bool) TaskResult {
return r
}
t.Verification.Mode = strings.TrimSpace(t.Verification.Mode)
t.Verification.TaskType = normalizeTaskType(t.Verification.TaskType)
t.Verification.EvidenceGrade = normalizeEvidenceGrade(t.Verification.Mode, t.Verification.EvidenceGrade)
r.TaskType = t.Verification.TaskType
r.EvidenceGrade = t.Verification.EvidenceGrade
if validationErr := validateVerification(t.Verification); validationErr != "" {
r.Verified = false
r.Reason = validationErr
return r
}
if t.Verification.Command == "" {
r.Reason = "verification.command is empty"
r.Verified = false
@@ -283,6 +360,8 @@ func verifyTask(t taskEntry, dryRun bool) TaskResult {
r.Stdout = stdout.String()
r.Stderr = stderr.String()
r.StdoutSummary = summarizeOutput(r.Stdout)
r.StderrSummary = summarizeOutput(r.Stderr)
if r.ExitCode != 0 && t.Verification.Mode == "test_pass" {
r.Verified = false
@@ -325,3 +404,134 @@ func verifyTask(t taskEntry, dryRun bool) TaskResult {
return r
}
func normalizeEvidenceGrade(mode, explicit string) string {
if explicit = strings.TrimSpace(explicit); explicit != "" {
return explicit
}
switch strings.TrimSpace(mode) {
case "test_pass":
return "runtime-verified"
case "artifact_present":
return "artifact-present"
case "semantic":
return "doc-claimed"
default:
return ""
}
}
func normalizeTaskType(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return "unspecified"
}
return raw
}
func normalizeStatusFromText(raw string) string {
lower := strings.ToLower(strings.TrimSpace(raw))
switch {
case strings.Contains(raw, "✅") || strings.Contains(raw, "完成"):
return "completed"
case strings.Contains(raw, "🟡") || strings.Contains(raw, "进行中"):
return "in_progress"
case strings.Contains(raw, "🔶") || strings.Contains(raw, "🔴") || strings.Contains(raw, "待启动") || strings.Contains(raw, "未开始"):
return "planned"
case strings.Contains(raw, "⏸️") || strings.Contains(raw, "待规划") || strings.Contains(raw, "暂停"):
return "paused"
case lower == "":
return "unknown"
default:
return "unknown"
}
}
func filterTasksByStatus(tasks []taskEntry, filter string) ([]taskEntry, error) {
filter = strings.TrimSpace(filter)
if filter == "" {
filter = "all"
}
valid := map[string]struct{}{
"all": {},
"completed": {},
"in_progress": {},
"planned": {},
"paused": {},
"unknown": {},
}
if _, ok := valid[filter]; !ok {
return nil, fmt.Errorf("unsupported status filter: %s", filter)
}
if filter == "all" {
return tasks, nil
}
filtered := make([]taskEntry, 0, len(tasks))
for _, t := range tasks {
status := t.Status
if status == "" {
status = "unknown"
}
if status == filter {
filtered = append(filtered, t)
}
}
return filtered, nil
}
func summarizeOutput(raw string) string {
cleaned := strings.TrimSpace(raw)
if cleaned == "" {
return ""
}
cleaned = strings.Join(strings.Fields(cleaned), " ")
const limit = 220
if len(cleaned) <= limit {
return cleaned
}
return cleaned[:limit] + "..."
}
func validateVerification(v Verification) string {
validModes := map[string]struct{}{
"test_pass": {},
"artifact_present": {},
"semantic": {},
}
if _, ok := validModes[v.Mode]; !ok {
return fmt.Sprintf("unsupported verification mode: %s", v.Mode)
}
validGrades := map[string]struct{}{
"runtime-verified": {},
"artifact-present": {},
"doc-claimed": {},
}
if v.EvidenceGrade != "" {
if _, ok := validGrades[v.EvidenceGrade]; !ok {
return fmt.Sprintf("unsupported evidence grade: %s", v.EvidenceGrade)
}
}
validTaskTypes := map[string]struct{}{
"unspecified": {},
"code": {},
"automation": {},
"documentation": {},
"configuration": {},
"data": {},
"analysis": {},
}
if _, ok := validTaskTypes[v.TaskType]; !ok {
return fmt.Sprintf("unsupported task type: %s", v.TaskType)
}
if (v.TaskType == "code" || v.TaskType == "automation") && v.Mode == "semantic" {
return fmt.Sprintf("semantic-only verification is not allowed for %s tasks", v.TaskType)
}
return ""
}