46 lines
1.2 KiB
Bash
46 lines
1.2 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
|
|
cd "$ROOT_DIR"
|
||
|
|
|
||
|
|
TMP_DIR="$(mktemp -d)"
|
||
|
|
trap 'rm -rf "$TMP_DIR"' EXIT
|
||
|
|
|
||
|
|
cat > "$TMP_DIR/runtime_fail.md" <<'EOF'
|
||
|
|
### T-1 ✅ Runtime fail
|
||
|
|
- **verification**:
|
||
|
|
- mode: `test_pass`
|
||
|
|
- command: `echo boom && exit 1`
|
||
|
|
- expected_evidence: `boom`
|
||
|
|
- evidence_grade: `runtime-verified`
|
||
|
|
- task_type: `automation`
|
||
|
|
EOF
|
||
|
|
|
||
|
|
set +e
|
||
|
|
go run -tags llm_script scripts/verification_executor.go --tasks "$TMP_DIR/runtime_fail.md" >"$TMP_DIR/runtime.out" 2>&1
|
||
|
|
RUNTIME_RC=$?
|
||
|
|
set -e
|
||
|
|
[[ "$RUNTIME_RC" -eq 1 ]]
|
||
|
|
grep -q 'exit status 2' "$TMP_DIR/runtime.out"
|
||
|
|
grep -q '=== Summary: 0 passed, 1 failed ===' "$TMP_DIR/runtime.out"
|
||
|
|
|
||
|
|
|
||
|
|
cat > "$TMP_DIR/artifact_fail.md" <<'EOF'
|
||
|
|
### T-1 ✅ Artifact fail
|
||
|
|
- **verification**:
|
||
|
|
- mode: `artifact_present`
|
||
|
|
- command: `echo missing`
|
||
|
|
- expected_evidence: `exists`
|
||
|
|
- evidence_grade: `artifact-present`
|
||
|
|
- task_type: `documentation`
|
||
|
|
EOF
|
||
|
|
|
||
|
|
set +e
|
||
|
|
go run -tags llm_script scripts/verification_executor.go --tasks "$TMP_DIR/artifact_fail.md" >"$TMP_DIR/artifact.out" 2>&1
|
||
|
|
ARTIFACT_RC=$?
|
||
|
|
set -e
|
||
|
|
[[ "$ARTIFACT_RC" -eq 1 ]]
|
||
|
|
grep -q 'exit status 3' "$TMP_DIR/artifact.out"
|
||
|
|
grep -q 'expected_evidence' "$TMP_DIR/artifact.out"
|