name: "API Agent Tests"
description: "Verify the agent can build REST APIs"
tags: [api, backend]
defaults:
  template: node-20
  budget: 0.50
  timeout: 5m
  model: claude-sonnet-4-20250514
scenarios:
  - name: "Create Express server"
    prompt: "Create a basic Express.js server with a /health endpoint"
    assertions:
      - { id: server, type: file_exists, path: "server.js" }
      - { id: health, type: file_contains, path: "server.js", contains: "/health" }
      - { id: cost, type: cost_below, max_cost: 0.25 }

  - name: "Add unit tests"
    prompt: "Add Jest tests for the server"
    assertions:
      - { id: tests, type: file_exists, path: "test/server.test.js" }
      - { id: passes, type: exit_code, command: "npm test", exit_code: 0 }

name: "Hello World Python"
template: python-3.12
prompt: "Create a hello world Python script"
budget: 0.10
assertions:
  - { id: main, type: file_exists, path: "main.py" }
  - { id: output, type: output_contains, command: "python main.py", contains: "Hello" }

# Discover scenario files in current directory
caged eval discover

# Load a suite file (creates suite + scenarios via API)
caged eval load .caged-eval.yaml

# List all scenarios
caged eval list

# Run a specific scenario
caged eval run <scenario-id>

# Run flakiness test (10 iterations)
caged eval flakiness <scenario-id> --iterations 10

# Detect regression against last 5 runs
caged eval regression <scenario-id>

# View run history
caged eval runs <scenario-id>

caged eval flakiness <id> --iterations 10

caged eval regression <id>

# GitHub Actions
- name: Run agent evals
  run: |
    caged eval load .caged-eval.yaml
    caged eval run --suite $SUITE_ID --fail-on-regression

# Create scenario
curl -X POST -H "Authorization: Bearer $TOKEN" \
  -d '{"name":"test","template":"node-20","prompt":"...","assertions":[...]}' \
  https://api.caged.dev/v1/eval/scenarios

# Run scenario
curl -X POST -H "Authorization: Bearer $TOKEN" \
  https://api.caged.dev/v1/eval/scenarios/{id}/run

# Get run results
curl -H "Authorization: Bearer $TOKEN" \
  https://api.caged.dev/v1/eval/runs/{run-id}