diff --git a/README.md b/README.md index b2647cc..7780ee1 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,23 @@ npm test --workspace=@void-server/incident-fingerprint npm test --workspace=@void-server/incident-formation ``` +### 5. Issue Agent E2E Monitoring +```bash +# Requires GOOGLE_API_KEY in .env + +# Install dependencies first +pip install -e packages/evaluator -e packages/issue-agent + +# Run all 14 scenarios (produces full report per scenario) +packages/evaluator/.venv/bin/python3 packages/issue-agent/tests/e2e_monitor.py + +# Run specific scenario(s) +packages/evaluator/.venv/bin/python3 packages/issue-agent/tests/e2e_monitor.py example tool-anomaly + +# Run issue agent unit tests +PYTHONPATH=packages/issue-agent/src:packages/evaluator/src:packages/issue-agent/tests python3 -m unittest packages/issue-agent/tests/test_schemas.py packages/issue-agent/tests/test_agent.py packages/issue-agent/tests/test_repository.py packages/issue-agent/tests/test_mapper.py -v +``` + --- ## 🧩 Packages diff --git a/packages/issue-agent/README.md b/packages/issue-agent/README.md new file mode 100644 index 0000000..cb5d239 --- /dev/null +++ b/packages/issue-agent/README.md @@ -0,0 +1,286 @@ +# VOID Issue Agent + +Converts an evaluated AI incident into an engineering-ready GitHub Issue. + +The Evaluator answers *"What happened?"* β€” the Issue Agent answers *"Where is the likely bug and what should engineering investigate first?"* + +--- + +## Logic Flow (End to End) + +``` +Incident Snapshot (JSON) + β”‚ + β–Ό + Evidence Extractor ── extracts failed tool calls per failure mode + β”‚ + β–Ό + PydanticAI Agent ── 4 tools: search_repo, read_file, build_code_graph, create_issue + β”‚ + β”œβ”€β”€ Repository Investigator (search β†’ graph β†’ read β†’ analyze) + └── Report Writer (structured EngineeringReport) + β”‚ + β–Ό + Engineering Report (JSON) + β”‚ + β–Ό + GitHub Issue (skipped in dev mode) +``` + +### Step 1: Load & Parse Incident Snapshot + +The entry point (`__main__.py`) reads JSON from stdin, validates it into an `IncidentSnapshot`: + +```python +raw = json.loads(sys.stdin.read()) +snapshot = IncidentSnapshot.model_validate(raw) +``` + +The snapshot contains the execution trace (agent steps + tool calls), evaluation result (failure modes, severity, confidence), and telemetry. + +### Step 2: Extract Evidence (`evidence.py`) + +Walks every failure mode and collects failed tool calls from the trace: + +```python +def extract_evidence(snapshot: IncidentSnapshot) -> list[Evidence]: + for fm in snapshot.evaluation.failure_modes: + trace_lines = [] + for step in snapshot.execution_trace.agent_steps: + for tc in step.tool_calls: + if not tc.success: + trace_lines.append(f"Tool '{tc.name}' failed: {tc.error or 'unknown error'}") +``` + +Output: one `Evidence` per failure mode β€” failure mode name, summary, supporting trace lines, confidence score. + +### Step 3: Build Agent Input + +The `run_issue_agent()` function in `agent.py` serializes the snapshot + evidence into a single JSON blob the LLM receives as the user prompt: + +```python +input_data = { + "incident_id": snapshot.incident_id, + "failure_modes": snapshot.evaluation.failure_modes, + "confidence": snapshot.evaluation.confidence, + "reasoning": snapshot.evaluation.reasoning, + "severity": snapshot.evaluation.severity, + "evidence": [e.model_dump() for e in evidence], + "timeline": [t.model_dump() for t in timeline], + "agent_steps": [{"step_type": s.step_type, "tool_calls": ..., "planner_output": s.planner_output, "context": s.context, "latency_ms": s.latency_ms}], + "model": snapshot.execution_trace.model, + "tokens_used": snapshot.execution_trace.tokens_used, + "total_latency_ms": snapshot.execution_trace.total_latency_ms, + "telemetry": snapshot.telemetry, + "metadata": snapshot.metadata, +} +``` + +### Step 4: LLM Agent Loop (`agent.py`) + +A single `Agent` with typed `EngineeringReport` output and 4 tools: + +```python +agent = Agent( + model="google:gemini-3.1-flash-lite", + output_type=EngineeringReport, + system_prompt="You are a senior engineer investigating an AI system incident...", +) +``` + +The system prompt instructs the LLM to: +1. Review the incident snapshot +2. Use repository tools to find relevant code +3. Search for symbols/filenames from the trace +4. Build code graphs around suspected files +5. Read only the functions needed to understand the bug +6. Write an **EngineeringReport** with all fields populated + +**Tools registered on the agent:** + +| Tool | Purpose | +|---|---| +| `search_repo(query)` | Find files matching a symbol/name | +| `read_file(path)` | Read file contents from the repo (truncated at 50KB) | +| `build_code_graph(file_paths)` | Parse imports β†’ dependency graph | + +**Dev Mode:** Set `VOID_DEV_MODE=1` β€” `create_github_issue` tool is not registered, agent returns the report as JSON only. The repo also switches from `GitHubRepo` (GitHub API) to `LocalRepo` (local filesystem): + +```python +repo = LocalRepo() if os.environ.get("VOID_DEV_MODE") == "1" else GitHubRepo() +``` + +**Retry logic:** Catches `ModelHTTPError(status_code=429)` from Gemini's rate limiter. Parses `retryDelay` from the error body (nested in `error.details[*].retryDelay`), sleeps, retries up to 5 times. Uses the server's requested delay (or 20s fallback) β€” not exponential backoff. + +**Structured logging:** Every phase of the agent run is logged as structured events: +- `system_prompt` / `user_prompt` β€” what the LLM received +- `model_response` β€” free-text LLM output (truncated to 500 chars in logs) +- `tool_call` / `tool_return` β€” tool name, arg count, status only (no args/body to avoid sensitive data leakage) +- `run_complete` β€” tokens, latency, retries, files_read, confidence + +### Step 5: Repository Access (`repository.py`) + +Two implementations: + +**`GitHubRepo`** β€” uses GitHub PAT from `GITHUB_TOKEN` env var, calls GitHub REST API: +- `search_symbol(symbol)` β€” recursive tree search for files matching the symbol, with content-aware matching +- `read_file(path)` β€” GET `/contents/{path}` returns base64-decoded file content +- `build_code_graph(file_paths)` β€” AST-based import parsing into `CodeGraph` +- `create_issue(title, body)` β€” POST `/issues` to create a real GitHub issue + +**`LocalRepo`** β€” reads from local filesystem, `create_issue()` returns `None`: +```python +class LocalRepo: + def search_symbol(self, symbol): + # rglob for files matching symbol, with content-aware matching + def read_file(self, path): + # read local file with path traversal protection, returns None if not found + def build_code_graph(self, file_paths): + # AST-based import parsing β†’ CodeGraph + def create_issue(self, title, body): + return None +``` + +**Code graph** is built by AST-parsing imports and function/class definitions, resolving against known repo files to avoid stdlib/phantom dependencies: + +```python +# AST parses "from foo import bar" + "import baz" +# Creates edges: tool_timeout.py ─imports─► agent.py +# Also indexes function defs as graph nodes: tool_timeout.py::search_code() +``` + +### Step 6: Engineering Report (`schemas.py`) + +The agent's typed output (18 fields): + +```python +class EngineeringReport(BaseModel): + summary: str # one-line summary of the bug + root_cause: str # root cause analysis + evidence: list[str] # supporting evidence strings + suspected_components: list[str] # e.g. ["Planner", "ToolExecutor"] + relevant_files: list[str] # file paths + relevant_functions: list[str] # function names + suggested_investigation: list[str] # next steps for engineering + suggested_fix: str # proposed fix description + suggested_tests: list[str] # test scenarios to add + confidence: float # 0.0 - 1.0 + + executive_summary: str = "" # high-level incident description + impact: str = "" # what broke and for whom + timeline: list[TimelineEvent] = [] # ordered reconstruction of incident + repository_findings: RepositoryFindings = Field(default_factory=RepositoryFindings) + missing_context: MissingContext | None = None + evidence_analysis: str = "" # how each conclusion is grounded + secondary_effects: list[str] = [] # cascading failures + issue_title: str = "" # ready-to-use GitHub issue title +``` + +### Step 7: GitHub Issue Creation (Host Code) + +After the agent returns the report, host code (`__main__.py` or pipeline) calls `create_github_issue_from_report()` to create exactly one GitHub issue with the full report β€” avoiding the model skipping/repeating the tool or 429 replays creating duplicates. + +--- + +## Architecture + +``` +packages/issue-agent/src/issue_agent/ +β”œβ”€β”€ __init__.py +β”œβ”€β”€ __main__.py # CLI entry: read stdin β†’ run β†’ print report +β”œβ”€β”€ agent.py # PydanticAI Agent, tools, retry, logging +β”œβ”€β”€ evidence.py # Failed tool call extraction +β”œβ”€β”€ repository.py # GitHubRepo (prod) / LocalRepo (dev) +└── schemas.py # Pydantic models: IncidentSnapshot β†’ EngineeringReport +``` + +--- + +## Production vs Dev Mode + +| Aspect | Production | Dev Mode | +|---|---|---| +| Env var | (unset) | `VOID_DEV_MODE=1` | +| Repo backend | `GitHubRepo` (GitHub API) | `LocalRepo` (local filesystem) | +| Issue creation | Host code calls `create_github_issue_from_report()` after validation | No issue created | +| Report output | Printed to stdout + posted to GitHub | Printed to stdout only | + +--- + +## Usage + +```bash +# Install +pip install -e packages/issue-agent + +# Configure (production) +export GITHUB_TOKEN=ghp_... +export DEMO_REPOSITORY=owner/repo + +# Run (reads incident JSON from stdin) +cat evaluation_dataset/incidents/example/incident.json | PYTHONPATH=packages/issue-agent/src python -m issue_agent + +# Dev mode (no GitHub API calls) +VOID_DEV_MODE=1 cat evaluation_dataset/incidents/example/incident.json | PYTHONPATH=packages/issue-agent/src python -m issue_agent +``` + +Note: `VOID_DEV_MODE=1` must be passed to Python, not just `cat`. + +--- + +## E2E Monitoring + +A standalone script runs all 14 incident scenarios against the agent and prints a detailed report for each: + +```bash +# From repository root (requires GOOGLE_API_KEY in .env) +packages/evaluator/.venv/bin/python3 packages/issue-agent/tests/e2e_monitor.py + +# Or run specific scenarios +packages/evaluator/.venv/bin/python3 packages/issue-agent/tests/e2e_monitor.py example tool-anomaly +``` + +Output per scenario: +1. **Evaluator output** β€” classification, failure modes, severity, reasoning, recommendations +2. **LLM conversation** β€” system prompt, user prompt, every model response text, every tool call with args, tool return statuses +3. **Run metrics** β€” tokens (input/output/total), requests, tool calls, latency, retries, files read +4. **Engineering report** β€” all 18 fields from the structured output +5. **Summary table** β€” pass/fail per scenario with confidence scores + +Caveat: Gemini's free tier has 15 RPM quota β€” running all 14 scenarios in sequence will hit rate limits. Retry logic handles this, but some scenarios may still fail. Run individual scenarios or use a paid API key for batch runs. + +### Test Suite + +```bash +# Unit tests only (no LLM calls, fast) +PYTHONPATH=packages/issue-agent/src:packages/evaluator/src:packages/issue-agent/tests python -m unittest \ + tests/test_schemas.py tests/test_agent.py tests/test_repository.py tests/test_mapper.py -v +``` + +--- + +--- + +## Test Scenarios + +Each scenario under `tests/issue-agent/` (evaluation fixtures) and `evaluation_dataset/incidents/` (input incidents) contains: +- `incident.json` β€” full incident with traces + evaluator output +- `evaluation.json` β€” just the evaluation result +- `expected.json` β€” expected investigation targets for validation + +```text +evaluation_dataset/incidents/ +β”œβ”€β”€ tool_timeout/ # tool timed out twice, no retry logic +β”œβ”€β”€ planner_bug/ # sequential calls instead of batching +β”œβ”€β”€ handoff_failure/ # state lost during agent handoff +β”œβ”€β”€ context_overflow/ # document exceeded context window +└── tool_call_anomaly/ # null field propagated silently +``` + +Full scenario list: `context-overflow`, `handoff-failure`, `looping`, `silent-hallucination`, `tool-anomaly`, `crash-loop`, `critical-escalation`, `example`, `false-positive`, `insufficient-evidence`, `mixed-labels`, `rate-limit`, `recurring`, `transient-error`. + +--- + +## Monitor Limitations + +The monitor does not print every model response in full; structured logging truncates model text to 500 characters and prompts to 300. For complete conversation debugging, inspect the structured log output directly. \ No newline at end of file diff --git a/packages/issue-agent/pyproject.toml b/packages/issue-agent/pyproject.toml new file mode 100644 index 0000000..c673883 --- /dev/null +++ b/packages/issue-agent/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "issue-agent" +version = "0.1.0" +description = "VOID Issue Agent - converts evaluated AI incidents into engineering-ready GitHub Issues" +requires-python = ">=3.11" +dependencies = [ + "pydantic>=2.10.0", + "pydantic-ai>=0.0.14", + "httpx>=0.27.0", +] + +[project.scripts] +issue-agent = "issue_agent.__main__:main" + +[tool.setuptools.packages.find] +where = ["src"] \ No newline at end of file diff --git a/packages/issue-agent/src/issue_agent/__init__.py b/packages/issue-agent/src/issue_agent/__init__.py new file mode 100644 index 0000000..e7ad1ea --- /dev/null +++ b/packages/issue-agent/src/issue_agent/__init__.py @@ -0,0 +1,8 @@ +from issue_agent.schemas import ( + IncidentSnapshot, Evidence, InvestigationTarget, + CodeGraph, CodeGraphNode, CodeGraphEdge, + EngineeringReport, GitHubIssueInput, +) +from issue_agent.repository import GitHubRepo, LocalRepo +from issue_agent.evidence import extract_evidence +from issue_agent.agent import run_issue_agent \ No newline at end of file diff --git a/packages/issue-agent/src/issue_agent/__main__.py b/packages/issue-agent/src/issue_agent/__main__.py new file mode 100644 index 0000000..c512f7f --- /dev/null +++ b/packages/issue-agent/src/issue_agent/__main__.py @@ -0,0 +1,58 @@ +import json +import sys +import os +import logging +from pydantic import ValidationError +from issue_agent.schemas import IncidentSnapshot +from issue_agent.repository import GitHubRepo, LocalRepo +from issue_agent.agent import run_issue_agent, create_github_issue_from_report + + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") +logger = logging.getLogger(__name__) + + +def main(): + try: + raw = json.loads(sys.stdin.read()) + except json.JSONDecodeError as e: + logger.error(f"Invalid JSON input: {e}") + sys.exit(1) + + try: + snapshot = IncidentSnapshot.model_validate(raw) + except ValidationError as e: + logger.error(f"Invalid incident snapshot: {e}") + sys.exit(1) + + dev_mode = os.environ.get("VOID_DEV_MODE") == "1" + if not dev_mode: + token = os.environ.get("GITHUB_TOKEN") + repo_name = os.environ.get("DEMO_REPOSITORY") + if not token: + logger.error("Production mode requires GITHUB_TOKEN environment variable") + sys.exit(1) + if not repo_name: + logger.error("Production mode requires DEMO_REPOSITORY environment variable") + sys.exit(1) + + repo = LocalRepo() if dev_mode else GitHubRepo() + report = run_issue_agent(snapshot, repo) + if report: + if dev_mode: + print(report.model_dump_json(indent=2)) + else: + issue_url = create_github_issue_from_report(repo, report, snapshot.incident_id) + if issue_url: + logger.info(f"Created GitHub issue: {issue_url}") + print(f"GitHub issue created: {issue_url}") + else: + logger.error("Failed to create GitHub issue") + sys.exit(1) + else: + logger.error("Issue agent failed to produce a report") + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/packages/issue-agent/src/issue_agent/agent.py b/packages/issue-agent/src/issue_agent/agent.py new file mode 100644 index 0000000..4613f38 --- /dev/null +++ b/packages/issue-agent/src/issue_agent/agent.py @@ -0,0 +1,382 @@ +import json +import logging +import time +from typing import Any +from pydantic_ai import Agent, RunContext, ModelHTTPError + +from issue_agent.schemas import IncidentSnapshot, EngineeringReport, TimelineEvent, RepositoryFindings, RepositoryValidation, MissingContext +from issue_agent.evidence import extract_evidence + + +logger = logging.getLogger(__name__) +_log = logger.getChild("structured") + +_MAX_FILE_CHARS = 50000 + + +class IssueAgentDeps: + def __init__(self, repo: Any): + self.repo = repo + self.files_read: list[str] = [] + self.tool_calls_used: int = 0 + + +def _build_timeline(snapshot: IncidentSnapshot, evidence_list: list) -> list[TimelineEvent]: + timeline: list[TimelineEvent] = [] + + invisible_steps: set[int] = set() + for si, step in enumerate(snapshot.execution_trace.agent_steps): + timeline.append(TimelineEvent( + event_type="execution_step", + step_index=si, + description=f"Agent step {step.step_type}", + source="trace", + )) + for tc in step.tool_calls: + if not tc.success: + invisible_steps.add(si) + timeline.append(TimelineEvent( + event_type="tool_call", + step_index=si, + description=f"Tool '{tc.name}' failed: {tc.error or 'unknown error'}", + source="trace", + )) + else: + timeline.append(TimelineEvent( + event_type="tool_call", + step_index=si, + description=f"Tool '{tc.name}' succeeded", + source="trace", + )) + + for ei, ev in enumerate(evidence_list): + timeline.append(TimelineEvent( + event_type="evidence", + step_index=min(ev.step_indices) if getattr(ev, "step_indices", None) else None, + description=f"Evidence: {ev.failure_mode} β€” {ev.summary[:120]}", + source="trace", + evidence_refs=[ei], + )) + + for fi, fm in enumerate(snapshot.evaluation.failure_modes): + first_bad = min(invisible_steps) if invisible_steps else None + timeline.append(TimelineEvent( + event_type="failure_observable", + step_index=first_bad, + description=f"Evaluator detected failure mode: {fm}", + source="evaluator", + evidence_refs=[fi], + )) + + last_failed = max(invisible_steps) if invisible_steps else (len(snapshot.execution_trace.agent_steps) - 1) + timeline.append(TimelineEvent( + event_type="root_cause", + step_index=last_failed, + description=f"Final failure: {snapshot.evaluation.reasoning[:200]}", + source="evaluator", + )) + + return timeline + + +SYSTEM_PROMPT = """You are a senior engineer investigating an AI system incident. + +Your task: +1. Review the incident snapshot (execution traces, evaluator output, telemetry, timeline). +2. Use repository tools to find the relevant code and validate evaluator findings. +3. Search for symbols, filenames, and function names from the trace. +4. Build a code graph around suspected files and traverse it. +5. Read the smallest set of functions needed to understand the bug. +6. Write an engineering report with ALL fields populated. + +RULES: + +**Timeline Reconstruction** +- The timeline is ordered: execution steps and tool calls first, then evidence, then evaluator findings. +- Identify where the failure first became observable (look for failed tool calls in the timeline). +- Describe how it propagated and explain the final incorrect behavior. +- The engineer reading this should understand "where did the incident begin, and how did it propagate into the final failure?" + +**Evidence Grounding** +- For every major conclusion in root_cause, suspected_components, suggested_investigation, and suggested_fix, reference supporting evidence. +- Use evidence_analysis to explain how each conclusion is supported. +- If evidence is insufficient, state "INSUFFICIENT EVIDENCE" and explain what is missing. Do not speculate. + +**Repository Validation** +- The input includes "evaluator_suspected_components" β€” a list of components the evaluator identified. +- You MUST copy these directly into the `suspected_components` output field. This field is REQUIRED and MUST NOT BE EMPTY if evaluator provided components. +- Then validate each one by searching the repository. +- Populate BOTH: + 1. suspected_components β€” COPY the evaluator_suspected_components list here exactly (from evaluator + any you discover). THIS IS MANDATORY. + 2. repository_findings.validated_components β€” detailed validation status for each: + - "confirmed" β€” found in repository (include found_paths) + - "suggested" β€” mentioned by evaluator but not found + - "not_found" β€” searched but does not exist in this repo + - "not_searched" β€” not attempted +- Also populate repository_findings.files_found, functions_found, and symbols_searched. +- If no relevant implementation can be located, populate missing_context explaining why. + +**Hallucination Prevention** +- Never invent repository files, functions, classes, APIs, tools, services, or components. +- Every referenced entity must originate from either: evaluator output OR repository investigation (search_repo, read_file, build_code_graph). +- If no supporting evidence exists, state that the information could not be verified. + +**Root Cause Analysis** +- Explain: what failed, why it failed, what evidence supports this conclusion, what system component is responsible, and what secondary effects occurred. +- If multiple contributing factors exist, list them separately in secondary_effects. + +**Regression Tests** +- Generate tests that directly validate the identified failure mode. +- Map failure mode to test type: + - context_overflow -> token budget tests + - handoff_failure -> context propagation tests + - looping -> retry limit tests + - tool_anomaly -> tool failure handling tests + - hallucination -> grounding verification tests + - (other failure modes -> appropriate specific test type) +- Avoid generic testing recommendations. + +Fill ALL fields of the report: +executive_summary, impact, timeline (analyze the pre-built one), root_cause, evidence_analysis, evidence, suspected_components, repository_findings, missing_context, relevant_files, relevant_functions, suggested_investigation, suggested_fix, suggested_tests, secondary_effects, confidence, issue_title, summary. +""" + + +def _build_agent() -> Agent: + agent = Agent( + model="google:gemini-3.1-flash-lite", + deps_type=IssueAgentDeps, + output_type=EngineeringReport, + system_prompt=SYSTEM_PROMPT, + ) + + @agent.tool + def search_repo(ctx: RunContext[IssueAgentDeps], query: str) -> str: + """Search the repository for files matching a symbol, function, or filename. + + Args: + query: Symbol, function name, or filename pattern to search for. + + Returns: + JSON list of matches with path and match type (content/filename). + """ + result = ctx.deps.repo.search_symbol(query) + ctx.deps.tool_calls_used += 1 + return json.dumps(result, indent=2) + + @agent.tool + def read_file(ctx: RunContext[IssueAgentDeps], path: str) -> str: + """Read a file from the repository. + + Args: + path: Repository-relative file path to read. + + Returns: + File content (truncated at 50KB) or error message. + """ + content = ctx.deps.repo.read_file(path) + ctx.deps.files_read.append(path) + ctx.deps.tool_calls_used += 1 + if content is None: + return f"File not found: {path}" + if len(content) > _MAX_FILE_CHARS: + content = content[:_MAX_FILE_CHARS] + f"\n" + return content + + @agent.tool + def build_code_graph(ctx: RunContext[IssueAgentDeps], file_paths: list[str]) -> str: + """Build an import dependency graph for the given files. + + Args: + file_paths: List of repository-relative file paths to analyze. + + Returns: + JSON-serialized CodeGraph with nodes (files/functions/classes) and edges (imports). + """ + graph = ctx.deps.repo.build_code_graph(file_paths) + ctx.deps.tool_calls_used += 1 + return graph.model_dump_json(indent=2) + + return agent + + +def create_github_issue_from_report(repo: Any, report: EngineeringReport, incident_id: str) -> str | None: + title = report.issue_title or f"Incident Report: {report.summary[:80]}" + body_parts = [ + f"## Incident: {incident_id}", + f"**Confidence:** {report.confidence}", + f"**Impact:** {report.impact}", + f"**Summary:** {report.summary}", + f"**Root Cause:** {report.root_cause}", + ] + if report.evidence: + body_parts.append("### Evidence\n" + "\n".join(f"- {e}" for e in report.evidence)) + if report.suspected_components: + body_parts.append("### Suspected Components\n" + "\n".join(f"- {c}" for c in report.suspected_components)) + if report.suggested_investigation: + body_parts.append("### Suggested Investigation\n" + "\n".join(f"- {i}" for i in report.suggested_investigation)) + if report.suggested_fix: + body_parts.append(f"### Suggested Fix\n{report.suggested_fix}") + if report.suggested_tests: + body_parts.append("### Regression Tests\n" + "\n".join(f"- {t}" for t in report.suggested_tests)) + body = "\n\n".join(body_parts) + return repo.create_issue(title, body) + + +def run_issue_agent( + snapshot: IncidentSnapshot, + repo: Any, + max_retries: int = 5, +) -> EngineeringReport | None: + evidence = extract_evidence(snapshot) + deps = IssueAgentDeps(repo=repo) + + timeline = _build_timeline(snapshot, evidence) + + input_data = { + "incident_id": snapshot.incident_id, + "failure_modes": snapshot.evaluation.failure_modes, + "confidence": snapshot.evaluation.confidence, + "reasoning": snapshot.evaluation.reasoning, + "severity": snapshot.evaluation.severity, + "evidence": [e.model_dump() for e in evidence], + "timeline": [t.model_dump() for t in timeline], + "evaluator_suspected_components": snapshot.metadata.get("suspected_components", []), + "agent_steps": [ + { + "step_type": s.step_type, + "planner_output": s.planner_output, + "tool_calls": [t.model_dump() for t in s.tool_calls], + "context": s.context, + "latency_ms": s.latency_ms, + } + for s in snapshot.execution_trace.agent_steps + ], + "model": snapshot.execution_trace.model, + "tokens_used": snapshot.execution_trace.tokens_used, + "total_latency_ms": snapshot.execution_trace.total_latency_ms, + "telemetry": snapshot.telemetry, + "metadata": snapshot.metadata, + } + input_json = json.dumps(input_data, indent=2) + input_size = len(input_json) + + agent = _build_agent() + + last_error: Exception | None = None + start_time = time.monotonic() + + for attempt in range(max_retries): + _rate_limit_wait() + try: + result = agent.run_sync(input_json, deps=deps) + break + except ModelHTTPError as e: + last_error = e + if e.status_code != 429: + _log.error("model_http_error", extra={"status": e.status_code, "model": e.model_name}) + return None + retry_after = _parse_retry_delay(e.body) + _log.warning( + "rate_limit_retry", + extra={"attempt": attempt + 1, "max_retries": max_retries, "retry_after": retry_after}, + ) + if attempt + 1 == max_retries: + return None + time.sleep(retry_after) + except Exception as e: + last_error = e + _log.error("unexpected_error", extra={"error": str(e)}) + return None + else: + _log.error("max_retries_exceeded", extra={"max_retries": max_retries, "last_error": str(last_error)}) + return None + + elapsed = time.monotonic() - start_time + usage = result.usage + + _log.info( + "run_complete", + extra={ + "run_id": result.run_id, + "input_tokens": usage.input_tokens, + "output_tokens": usage.output_tokens, + "total_tokens": usage.total_tokens, + "requests": usage.requests, + "tool_calls": usage.tool_calls, + "latency_s": round(elapsed, 3), + "input_size_bytes": input_size, + "retries": attempt, + "files_read": list(deps.files_read), + "files_read_count": len(deps.files_read), + "tool_calls_used": deps.tool_calls_used, + "confidence": result.output.confidence, + "summary": result.output.summary[:120] if result.output.summary else "", + }, + ) + + for msg in result.all_messages(): + for part in getattr(msg, "parts", ()): + pk = getattr(part, "part_kind", None) + if pk == "system-prompt": + _log.info("system_prompt", extra={"content": part.content[:300]}) + elif pk == "user-prompt": + content = part.content[:300] if isinstance(part.content, str) else str(part.content)[:300] + _log.info("user_prompt", extra={"content": content}) + elif pk == "text": + _log.info("model_response", extra={"content": part.content[:500]}) + elif pk == "tool-call": + _log.info("tool_call", extra={"tool": part.tool_name, "tool_args_count": len(str(part.args))}) + elif pk == "tool-return": + status = "ok" if getattr(part, "outcome", None) in (None, "success") else "fail" + _log.info("tool_return", extra={"tool": part.tool_name, "status": status}) + + logger.info( + "Issue agent completed. Tokens: %d, Files read: %d, Confidence: %.2f", + deps.tool_calls_used, + len(deps.files_read), + result.output.confidence, + ) + return result.output + + +_last_request_time: float = 0.0 + + +def _rate_limit_wait(min_interval: float = 5.0): + global _last_request_time + now = time.monotonic() + elapsed = now - _last_request_time + if elapsed < min_interval: + time.sleep(min_interval - elapsed) + _last_request_time = time.monotonic() + + +def _parse_retry_delay(body: object, default: float = 10.0) -> float: + if isinstance(body, dict): + for key in ("retryDelay", "retry_delay", "Retry-After"): + raw = body.get(key) or _deep_get(body, key) + if raw is not None: + try: + if isinstance(raw, str) and raw.endswith("s"): + return float(raw[:-1]) + return float(raw) + except (ValueError, TypeError): + pass + return default * 2 + + +def _deep_get(d: dict, key: str) -> object | None: + if key in d: + return d[key] + for v in d.values(): + if isinstance(v, dict): + result = _deep_get(v, key) + if result is not None: + return result + elif isinstance(v, list): + for item in v: + if isinstance(item, dict): + result = _deep_get(item, key) + if result is not None: + return result + return None \ No newline at end of file diff --git a/packages/issue-agent/src/issue_agent/evidence.py b/packages/issue-agent/src/issue_agent/evidence.py new file mode 100644 index 0000000..485c1d9 --- /dev/null +++ b/packages/issue-agent/src/issue_agent/evidence.py @@ -0,0 +1,23 @@ +from issue_agent.schemas import IncidentSnapshot, Evidence + + +def extract_evidence(snapshot: IncidentSnapshot) -> list[Evidence]: + evidence_list = [] + for fm in snapshot.evaluation.failure_modes: + trace_lines = [] + for step in snapshot.execution_trace.agent_steps: + if step.tool_calls: + for tc in step.tool_calls: + if not tc.success: + trace_lines.append( + f"Tool '{tc.name}' failed: {tc.error or 'unknown error'}" + ) + evidence_list.append( + Evidence( + failure_mode=fm, + summary=snapshot.evaluation.reasoning, + supporting_trace=trace_lines or ["No specific trace captured"], + confidence=snapshot.evaluation.confidence, + ) + ) + return evidence_list \ No newline at end of file diff --git a/packages/issue-agent/src/issue_agent/repository.py b/packages/issue-agent/src/issue_agent/repository.py new file mode 100644 index 0000000..8c21487 --- /dev/null +++ b/packages/issue-agent/src/issue_agent/repository.py @@ -0,0 +1,201 @@ +import ast +import os +import re +import base64 +import httpx +from pathlib import Path +from issue_agent.schemas import CodeGraph, CodeGraphNode, CodeGraphEdge + + +GITHUB_API = "https://api.github.com" +_STDLIB_MODULES = {"os", "sys", "json", "re", "math", "time", "datetime", "collections", "typing", + "pathlib", "functools", "itertools", "logging", "hashlib", "base64", "copy", + "enum", "dataclasses", "abc", "io", "textwrap", "uuid", "importlib"} + + +def _ast_parse_code_graph(file_path: str, source: str, repo_files: set[str] | None = None) -> tuple[dict, list]: + nodes: dict[str, CodeGraphNode] = {file_path: CodeGraphNode(file_path=file_path, kind="file")} + edges: list[CodeGraphEdge] = [] + + for match in re.finditer(r'^(?:from\s+(\S+)\s+)?import\s+(\S+)', source, re.MULTILINE): + module = match.group(1) or match.group(2) + target = module.replace(".", "/") + ".py" + if target[:-3] in _STDLIB_MODULES or target in _STDLIB_MODULES: + continue + if repo_files is None or target in repo_files: + nodes.setdefault(target, CodeGraphNode(file_path=target, kind="file")) + edges.append(CodeGraphEdge(source=file_path, target=target, relation="imports")) + + try: + tree = ast.parse(source) + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + key = f"{file_path}::{node.name}" + nodes[key] = CodeGraphNode(file_path=file_path, kind="function", symbol=node.name) + elif isinstance(node, ast.ClassDef): + key = f"{file_path}::{node.name}" + nodes[key] = CodeGraphNode(file_path=file_path, kind="class", symbol=node.name) + except SyntaxError: + for match in re.finditer(r'\bdef\s+(\w+)\s*\(', source): + key = f"{file_path}::{match.group(1)}" + nodes[key] = CodeGraphNode(file_path=file_path, kind="function", symbol=match.group(1)) + + return nodes, edges + + +def _search_source_content(path: str, source: str | None, query: str) -> bool: + if not source: + return False + pattern = re.compile(re.escape(query), re.IGNORECASE) + return bool(pattern.search(source)) + + +class GitHubRepo: + def __init__(self, token: str | None = None, repo: str | None = None): + self.token = token or os.environ.get("GITHUB_TOKEN", "") + self.repo = repo or os.environ.get("DEMO_REPOSITORY", "") + self._client = httpx.Client( + base_url=f"{GITHUB_API}/repos/{self.repo}", + headers={"Authorization": f"Bearer {self.token}", "Accept": "application/vnd.github.v3+json"}, + timeout=30, + ) + + def _safe_get(self, url: str, params: dict | None = None) -> dict: + try: + resp = self._client.get(url, params=params) + if resp.is_error: + return {"error": f"GitHub API error {resp.status_code}"} + return resp.json() + except httpx.RequestError as e: + return {"error": f"GitHub request failed: {e}"} + + def search_symbol(self, symbol: str) -> dict: + content_data = self._safe_get( + f"{GITHUB_API}/search/code", + params={"q": f"{symbol} repo:{self.repo}"}, + ) + + tree_data = self._safe_get("/git/trees/HEAD?recursive=1") + + matches = [] + seen_paths: set[str] = set() + pattern = re.compile(re.escape(symbol), re.IGNORECASE) + + if "error" not in content_data: + for item in content_data.get("items", []): + path = item["path"] + seen_paths.add(path) + matches.append({"path": path, "matched_in": "content"}) + + if "error" not in tree_data: + for item in tree_data.get("tree", []): + if item.get("type") != "blob": + continue + path = item["path"] + if path in seen_paths: + continue + if pattern.search(path): + seen_paths.add(path) + matches.append({"path": path, "matched_in": "filename"}) + + result = {"matches": matches} + incomplete = content_data.get("incomplete_results", False) if "error" not in content_data else False + truncated = tree_data.get("truncated", False) if "error" not in tree_data else False + if incomplete or truncated: + result["truncated"] = True + return result + + def read_file(self, path: str) -> str | None: + data = self._safe_get(f"/contents/{path}") + if "error" in data: + return None + if isinstance(data, list): + return None + raw = data.get("content", "") + if not raw: + return None + try: + return base64.b64decode(raw).decode("utf-8") + except (UnicodeDecodeError, ValueError): + return None + + def search_files(self, filename: str) -> list[dict]: + data = self._safe_get(f"{GITHUB_API}/search/code", params={"q": f"filename:{filename} repo:{self.repo}"}) + if "error" in data: + return [] + return [{"path": item["path"], "name": item["name"]} for item in data.get("items", [])] + + def build_code_graph(self, file_paths: list[str]) -> CodeGraph: + all_nodes, all_edges = {}, [] + for path in file_paths: + content = self.read_file(path) + if content is None: + continue + nodes, edges = _ast_parse_code_graph(path, content) + all_nodes.update(nodes) + all_edges.extend(edges) + return CodeGraph(nodes=list(all_nodes.values()), edges=all_edges) + + def create_issue(self, title: str, body: str) -> str | None: + try: + resp = self._client.post("/issues", json={"title": title, "body": body}) + if resp.is_error: + return None + return resp.json().get("html_url") + except httpx.RequestError: + return None + + +class LocalRepo: + def __init__(self, base_path: str | None = None): + self.base_path = Path(base_path or os.environ.get("LOCAL_REPO_PATH", ".")).resolve() + + def _resolve_path(self, path: str) -> Path | None: + candidate = (self.base_path / path).resolve() + try: + candidate.relative_to(self.base_path) + except ValueError: + return None + return candidate + + def search_symbol(self, symbol: str) -> dict: + pattern = re.compile(re.escape(symbol), re.IGNORECASE) + matches = [] + for f in self.base_path.rglob("*"): + if not f.is_file(): + continue + rel = f.relative_to(self.base_path) + if pattern.search(str(rel)): + try: + content = f.read_text(encoding="utf-8", errors="replace") + if _search_source_content(str(rel), content, symbol): + matches.append({"path": str(rel), "matched_in": "content"}) + else: + matches.append({"path": str(rel), "matched_in": "filename"}) + except (OSError, ValueError): + matches.append({"path": str(rel), "matched_in": "filename"}) + return {"matches": matches} + + def read_file(self, path: str) -> str | None: + full = self._resolve_path(path) + if full is None or not full.exists() or not full.is_file(): + return None + try: + return full.read_text(encoding="utf-8") + except (UnicodeDecodeError, ValueError): + return None + + def build_code_graph(self, file_paths: list[str]) -> CodeGraph: + repo_files = {str(f.relative_to(self.base_path)) for f in self.base_path.rglob("*.py") if f.is_file()} + all_nodes, all_edges = {}, [] + for path in file_paths: + content = self.read_file(path) + if content is None: + continue + nodes, edges = _ast_parse_code_graph(path, content, repo_files) + all_nodes.update(nodes) + all_edges.extend(edges) + return CodeGraph(nodes=list(all_nodes.values()), edges=all_edges) + + def create_issue(self, title: str, body: str) -> str | None: + return None \ No newline at end of file diff --git a/packages/issue-agent/src/issue_agent/schemas.py b/packages/issue-agent/src/issue_agent/schemas.py new file mode 100644 index 0000000..191035a --- /dev/null +++ b/packages/issue-agent/src/issue_agent/schemas.py @@ -0,0 +1,130 @@ +from pydantic import BaseModel, Field +from typing import Literal + + +class ToolCall(BaseModel): + name: str + input: str | None = None + output: str | None = None + latency_ms: float | None = None + success: bool + error: str | None = None + + +class AgentStep(BaseModel): + step_type: str + planner_output: str | None = None + tool_calls: list[ToolCall] = Field(default_factory=list) + context: str | None = None + latency_ms: float | None = None + + +class ExecutionTrace(BaseModel): + agent_steps: list[AgentStep] = Field(default_factory=list) + model: str | None = None + total_latency_ms: float | None = None + tokens_used: int | None = None + + +class EvaluationResult(BaseModel): + failure_modes: list[str] = Field(default_factory=list) + confidence: float = Field(ge=0.0, le=1.0) + reasoning: str = "" + urgency_tier: Literal["P0", "P1", "P2", "DEFER"] = "P2" + severity: Literal["CRITICAL", "HIGH", "MEDIUM", "LOW"] = "MEDIUM" + + +class IncidentSnapshot(BaseModel): + incident_id: str + execution_trace: ExecutionTrace + evaluation: EvaluationResult + telemetry: dict = Field(default_factory=dict) + metadata: dict = Field(default_factory=dict) + + +class Evidence(BaseModel): + failure_mode: str + summary: str + supporting_trace: list[str] = Field(default_factory=list) + confidence: float = Field(ge=0.0, le=1.0) + + +class InvestigationTarget(BaseModel): + file_path: str = "" + symbol: str = "" + reason: str = "" + + +class CodeGraphNode(BaseModel): + file_path: str + kind: Literal["file", "class", "function", "symbol"] = "file" + symbol: str | None = None + + +class CodeGraphEdge(BaseModel): + source: str + target: str + relation: Literal["imports", "calls", "inherits", "references"] + + +class CodeGraph(BaseModel): + nodes: list[CodeGraphNode] = Field(default_factory=list) + edges: list[CodeGraphEdge] = Field(default_factory=list) + + +class TimelineEvent(BaseModel): + event_type: Literal["execution_step", "tool_call", "failure_observable", "evidence", "root_cause"] + step_index: int | None = None + description: str + source: Literal["evaluator", "repository", "trace"] = "trace" + evidence_refs: list[int] = Field(default_factory=list) + + +class RepositoryValidation(BaseModel): + component: str + status: Literal["confirmed", "suggested", "not_found", "not_searched"] + found_paths: list[str] = Field(default_factory=list) + notes: str = "" + + +class RepositoryFindings(BaseModel): + validated_components: list[RepositoryValidation] = Field(default_factory=list) + files_found: list[str] = Field(default_factory=list) + functions_found: list[str] = Field(default_factory=list) + symbols_searched: list[str] = Field(default_factory=list) + missing_context_reason: str = "" + + +class MissingContext(BaseModel): + reason: str + missing_information: list[str] = Field(default_factory=list) + recommendations: list[str] = Field(default_factory=list) + + +class EngineeringReport(BaseModel): + summary: str + root_cause: str + evidence: list[str] = Field(default_factory=list) + suspected_components: list[str] = Field(default_factory=list) + relevant_files: list[str] = Field(default_factory=list) + relevant_functions: list[str] = Field(default_factory=list) + suggested_investigation: list[str] = Field(default_factory=list) + suggested_fix: str + suggested_tests: list[str] = Field(default_factory=list) + confidence: float = Field(ge=0.0, le=1.0) + + executive_summary: str = "" + impact: str = "" + timeline: list[TimelineEvent] = Field(default_factory=list) + repository_findings: RepositoryFindings = Field(default_factory=RepositoryFindings) + missing_context: MissingContext | None = None + evidence_analysis: str = "" + secondary_effects: list[str] = Field(default_factory=list) + issue_title: str = "" + + +class GitHubIssueInput(BaseModel): + owner: str + repo: str + title: str + body: str \ No newline at end of file diff --git a/packages/issue-agent/tests/__init__.py b/packages/issue-agent/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/issue-agent/tests/e2e_monitor.py b/packages/issue-agent/tests/e2e_monitor.py new file mode 100644 index 0000000..e61bd73 --- /dev/null +++ b/packages/issue-agent/tests/e2e_monitor.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""E2E monitoring script β€” runs all scenarios and prints a full report. +Not a test. Logs LLM responses, tool calls, retries, tokens, latency, final report.""" + +import json +import logging +import os +import sys +import time + +_HERE = os.path.dirname(__file__) +sys.path.insert(0, os.path.join(_HERE, "..", "src")) +sys.path.insert(0, os.path.join(_HERE, "..", "..", "evaluator", "src")) + +from issue_agent.agent import run_issue_agent +from issue_agent.repository import LocalRepo +from issue_agent.schemas import EngineeringReport + +# ── logging setup: capture structured logs with extra fields ────────────── +_log = logging.getLogger("issue_agent") +_log.setLevel(logging.DEBUG) + +captured: list[dict] = [] + + +class CaptureHandler(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + extras = { + k: v for k, v in record.__dict__.items() + if k not in logging.LogRecord("n", 0, "", 0, "", (), None).__dict__ + } + captured.append({"level": record.levelname, "msg": record.getMessage(), "extra": extras}) + + +_log.handlers.clear() +_log.addHandler(CaptureHandler()) + +# silence noisy libs +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("httpcore").setLevel(logging.WARNING) + +# ── helpers ─────────────────────────────────────────────────────────────── + +_PROJECT_ROOT = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) +SCENARIOS = [ + "context-overflow", "handoff-failure", "looping", "silent-hallucination", "tool-anomaly", + "crash-loop", "critical-escalation", "example", "false-positive", "insufficient-evidence", + "mixed-labels", "rate-limit", "recurring", "transient-error", +] + + +def load_json(path: str) -> dict: + with open(path) as f: + return json.load(f) + + +def run_evaluator(incident: dict) -> dict: + from evaluator.agent import Agent as EvaluatorAgent + agent = EvaluatorAgent() + result = agent.evaluate(incident) + if result is None: + return {"evaluation": {}, "metadata": {}} + return json.loads(result.model_dump_json()) + + +def build_snapshot(incident: dict, evaluator: dict): + from issue_agent.schemas import IncidentSnapshot, ExecutionTrace, AgentStep, ToolCall, EvaluationResult + eval_data = evaluator.get("evaluation", {}) + failure_modes = eval_data.get("failure_modes", []) + if not failure_modes: + failure_modes = incident.get("latest_labels", ["NONE_DETECTED"]) + steps = [] + for s in incident.get("agent_steps", []): + llm = s.get("llm_response", {}) + tool_calls = [ + ToolCall( + name=tc.get("name", ""), + input=tc.get("input", ""), + output=tc.get("output"), + latency_ms=tc.get("latency_ms"), + success=tc.get("success", True), + error=tc.get("error"), + ) + for tc in s.get("tool_calls", []) + ] + steps.append(AgentStep( + step_type=str(s.get("step_number", "")), + planner_output=llm.get("response", ""), + tool_calls=tool_calls, + context="", + latency_ms=s.get("telemetry", {}).get("latency_ms") if isinstance(s.get("telemetry"), dict) else None, + )) + telemetry = incident.get("telemetry", {}) + return IncidentSnapshot( + incident_id=incident.get("id", incident.get("execution_id", "unknown")), + execution_trace=ExecutionTrace( + agent_steps=steps, + model="gemini-3.1-flash-lite", + total_latency_ms=telemetry.get("total_latency_ms"), + tokens_used=telemetry.get("total_prompt_tokens", 0) or 0, + ), + evaluation=EvaluationResult( + failure_modes=failure_modes, + confidence=eval_data.get("confidence", 0.5), + reasoning=eval_data.get("suspected_root_cause", eval_data.get("summary", "")), + severity=_extract_severity(eval_data), + ), + telemetry=telemetry, + metadata={ + "classification": eval_data.get("classification", ""), + "summary": eval_data.get("summary", ""), + "suspected_components": eval_data.get("suspected_components", []), + "recommendations": eval_data.get("recommendations", []), + }, + ) + + +_SEVERITY_MAP = {"P0": "CRITICAL", "P1": "HIGH", "P2": "MEDIUM", "DEFER": "LOW"} + +def _extract_severity(eval_data: dict) -> str: + raw = eval_data.get("severity", eval_data.get("urgency_tier", eval_data.get("urgency", "MEDIUM"))) + if isinstance(raw, dict): + raw = raw.get("tier", str(raw.get("status", "MEDIUM"))) + mapped = _SEVERITY_MAP.get(raw) + if mapped: + return mapped + if isinstance(raw, str) and raw in ("CRITICAL", "HIGH", "MEDIUM", "LOW"): + return raw + return "MEDIUM" + + +def _load_dotenv(): + dotenv = os.path.join(_PROJECT_ROOT, ".env") + if not os.path.exists(dotenv): + return + with open(dotenv) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, val = line.split("=", 1) + key = key.strip() + val = val.strip().strip('"').strip("'") + if key and not os.environ.get(key): + os.environ[key] = val + + +_load_dotenv() + +# ── report printing ─────────────────────────────────────────────────────── + +SEP = "=" * 72 +SUB = "-" * 72 + + +def print_report(scenario: str) -> dict: + incident_path = os.path.join(_PROJECT_ROOT, "evaluation_dataset", "incidents", scenario, "incident.json") + scenario_dir = os.path.join(_HERE, "issue-agent", scenario) + eval_path = os.path.join(scenario_dir, "evaluation.json") + + incident = load_json(incident_path) + + # ── evaluator ────────────────────────────────────────────────────── + has_api = bool(os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")) + if has_api and os.path.exists(eval_path): + evaluator_output = load_json(eval_path) + elif has_api: + print(" [running evaluator...]") + evaluator_output = run_evaluator(incident) + os.makedirs(scenario_dir, exist_ok=True) + with open(eval_path, "w") as f: + json.dump(evaluator_output, f, indent=2) + else: + evaluator_output = {"evaluation": {}, "metadata": {}} + + eval_data = evaluator_output.get("evaluation", {}) + print(f"\n{SEP}") + print(f" SCENARIO: {scenario}") + print(f" ID: {incident.get('id', incident.get('execution_id', '?'))}") + print(f"{SEP}") + + print(f"\n ── EVALUATOR OUTPUT ──") + print(f" Classification: {eval_data.get('classification', '?')}") + print(f" Confidence: {eval_data.get('confidence', '?')}") + print(f" Failure modes: {', '.join(eval_data.get('failure_modes', []))}") + print(f" Severity: {_extract_severity(eval_data)}") + print(f"\n Summary:") + for line in eval_data.get("summary", "").split(". "): + print(f" β€’ {line.strip()}.") + print(f"\n Suspected root cause:") + for line in eval_data.get("suspected_root_cause", "").split(". "): + print(f" β€’ {line.strip()}.") + print(f"\n Suspected components: {', '.join(eval_data.get('suspected_components', []))}") + print(f"\n Reasoning:") + for r in eval_data.get("reasoning", []): + print(f" β€’ {r}") + print(f"\n Recommendations:") + for r in eval_data.get("recommendations", []): + print(f" β€’ {r}") + + # ── issue agent ──────────────────────────────────────────────────── + snapshot = build_snapshot(incident, evaluator_output) + repo = LocalRepo(os.path.join(_PROJECT_ROOT, "evaluation_dataset", "incidents", scenario)) + + captured.clear() + t0 = time.monotonic() + report: EngineeringReport | None = run_issue_agent(snapshot, repo, max_retries=5) + wall = time.monotonic() - t0 + logs = list(captured) + + # ── agent logs ───────────────────────────────────────────────────── + print(f"\n ── ISSUE AGENT LOGS ──") + + retry_events = [l for l in logs if l.get("msg") == "rate_limit_retry"] + if retry_events: + print(f" ⚠ RETRIES: {len(retry_events)}") + for r in retry_events: + e = r["extra"] + print(f" attempt {e.get('attempt', '?')}/{e.get('max_retries', '?')} retry_after={e.get('retry_after', '?')}s") + + error_events = [l for l in logs if l["level"] == "ERROR"] + if error_events: + print(f" βœ– ERRORS:") + for e in error_events: + print(f" {e['msg']}: {e['extra']}") + + # full LLM conversation + print("\n ── LLM CONVERSATION ──") + for log_entry in logs: + msg = log_entry["msg"] + ex = log_entry["extra"] + if msg == "system_prompt": + print(f"\n [SYSTEM PROMPT]\n{_indent(ex.get('content', ''), 4)}") + elif msg == "user_prompt": + print(f"\n [USER PROMPT]\n{_indent(ex.get('content', ''), 4)}") + elif msg == "model_response": + print(f"\n ── MODEL RESPONSE ──\n{_indent(ex.get('content', ''), 4)}") + elif msg == "tool_call": + print(f"\n ── TOOL CALL: {ex.get('tool', '?')} ──") + count = ex.get("tool_args_count") + if count is not None: + print(f" (args: {count} chars)") + elif msg == "tool_return": + status = ex.get("status", "?") + mark = "βœ“" if status == "ok" else "βœ—" + print(f" {mark} TOOL RETURN: {ex.get('tool', '?')} [{status}]") + elif msg == "run_complete": + print(f"\n ── RUN METRICS ──") + print(f" run_id: {ex.get('run_id', '?')}") + print(f" input_tokens: {ex.get('input_tokens', '?')}") + print(f" output_tokens:{ex.get('output_tokens', '?')}") + print(f" total_tokens: {ex.get('total_tokens', '?')}") + print(f" requests: {ex.get('requests', '?')}") + print(f" tool_calls: {ex.get('tool_calls', '?')}") + print(f" latency: {ex.get('latency_s', '?')}s") + print(f" retries: {ex.get('retries', '?')}") + print(f" files_read: {ex.get('files_read_count', 0)} {ex.get('files_read', [])}") + print(f" tool_calls: {ex.get('tool_calls_used', 0)}") + print(f" confidence: {ex.get('confidence', '?')}") + print(f" summary: {ex.get('summary', '')}") + + # ── final report ─────────────────────────────────────────────────── + print("\n ── ENGINEERING REPORT ──") + if report is None: + print(" βœ– Agent returned None (failed/rate-limited)") + else: + print(f" Issue Title: {report.issue_title or '(not set)'}") + print(f" Confidence: {report.confidence}") + print(f"\n Executive Summary: {report.executive_summary or report.summary}") + if report.impact: + print(f"\n Impact: {report.impact}") + print(f"\n ── Timeline ──") + if report.timeline: + for ev in report.timeline: + badge = { + "execution_step": "β–Ά", + "tool_call": "βš™", + "failure_observable": "βœ–", + "evidence": "β—ˆ", + "root_cause": "β—†", + }.get(ev.event_type, "β€’") + step = f" step#{ev.step_index}" if ev.step_index is not None else "" + refs = f" [evidence: {ev.evidence_refs}]" if ev.evidence_refs else "" + print(f" {badge} [{ev.source}]{step} {ev.description}{refs}") + else: + print(" (timeline not populated by agent)") + print(f"\n ── Root Cause ──") + print(f" {report.root_cause}") + if report.secondary_effects: + print(f"\n Secondary Effects:") + for s in report.secondary_effects: + print(f" β€’ {s}") + print(f"\n ── Evidence Analysis ──") + print(f" {report.evidence_analysis or '(not provided)'}") + print(f"\n Evidence:") + for e in report.evidence: + print(f" β€’ {e}") + print(f"\n Suspected Components: {', '.join(report.suspected_components)}") + print(f"\n ── Repository Findings ──") + rf = report.repository_findings + if rf.validated_components: + print(f" Validated Components:") + for vc in rf.validated_components: + paths = ", ".join(vc.found_paths) if vc.found_paths else "" + notes = f" β€” {vc.notes}" if vc.notes else "" + print(f" [{vc.status:12s}] {vc.component}{' ' + paths if paths else ''}{notes}") + else: + print(f" (no components validated)") + if rf.files_found: + print(f" Files Found: {', '.join(rf.files_found)}") + if rf.functions_found: + print(f" Functions Found: {', '.join(rf.functions_found)}") + if rf.symbols_searched: + print(f" Symbols Searched: {', '.join(rf.symbols_searched)}") + if rf.missing_context_reason: + print(f" Missing Context: {rf.missing_context_reason}") + if report.missing_context: + print(f"\n ── Missing Context ──") + print(f" Reason: {report.missing_context.reason}") + if report.missing_context.missing_information: + print(f" Missing Information:") + for m in report.missing_context.missing_information: + print(f" β€’ {m}") + if report.missing_context.recommendations: + print(f" Recommendations:") + for r in report.missing_context.recommendations: + print(f" β€’ {r}") + print(f"\n Relevant Files: {', '.join(report.relevant_files) if report.relevant_files else '(none)'}") + print(f"\n Relevant Functions: {', '.join(report.relevant_functions) if report.relevant_functions else '(none)'}") + print(f"\n Suggested Investigation:") + for i in report.suggested_investigation: + print(f" β€’ {i}") + print(f"\n Suggested Fix: {report.suggested_fix}") + print(f"\n Suggested Tests:") + for t in report.suggested_tests: + print(f" β€’ {t}") + + print(f"\n ── WALL CLOCK: {wall:.2f}s ──") + + return {"scenario": scenario, "logs": logs, "report": report} + + +def _indent(text: str, spaces: int = 4) -> str: + prefix = " " * spaces + return "\n".join(prefix + line for line in text.split("\n")) + + +# ── main ─────────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + targets = sys.argv[1:] if len(sys.argv) > 1 else SCENARIOS + + summary: list[dict] = [] + for s in targets: + r = print_report(s) + summary.append(r) + + print(f"\n{SEP}") + print(f" SUMMARY ({len(summary)} scenarios)") + print(f"{SEP}") + for r in summary: + ok = "βœ“" if r["report"] is not None else "βœ—" + if r["report"]: + conf = r["report"].confidence + print(f" {ok} {r['scenario']:30s} conf={conf:.2f}") + else: + print(f" {ok} {r['scenario']:30s} FAILED (no report)") \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/context-overflow/evaluation.json b/packages/issue-agent/tests/issue-agent/context-overflow/evaluation.json new file mode 100644 index 0000000..56b3602 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/context-overflow/evaluation.json @@ -0,0 +1,42 @@ +{ + "evaluation": { + "summary": "The agent exhibited a silent context overflow and hallucination, resulting in incorrect product specifications. The trace shows document truncation in Step 1 and a subsequent loss of context, leading the agent to provide false data in the final response.", + "classification": "REAL_INCIDENT", + "recoverability": "RECOVERABLE", + "confidence": 1.0, + "failure_modes": [ + "SILENT_CONTEXT_OVERFLOW", + "HALLUCINATION" + ], + "suspected_root_cause": "The agent's context window was overwhelmed by the retrieval of 50 documents, causing truncation of critical data (e.g., doc5) and subsequent degradation of reasoning accuracy.", + "suspected_components": [ + "retrieval_module", + "summarization_logic", + "context_manager" + ], + "reasoning": [ + "Step 1 shows document truncation (doc5 content truncated), indicating the retrieval volume exceeded the effective context window.", + "The final output (128GB, 4G, 300g) directly contradicts the retrieved content (256GB, 5G, 450g), confirming a hallucination.", + "The prompt token count increased significantly between steps, suggesting the agent struggled to maintain coherence as context grew.", + "The agent failed to reference the full document set, likely due to the truncation and context pressure." + ], + "recommendations": [ + "Implement a more aggressive document summarization or filtering step before passing data to the LLM.", + "Add a validation layer to check if retrieved content is truncated before proceeding to summarization.", + "Reduce the 'max_results' parameter in the retrieve_docs tool to fit within the safe context limit.", + "Implement a cross-check mechanism where the agent must cite the document ID for each claim in the final answer." + ], + "urgency": { + "tier": "P2", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The execution has terminated and the impact is contained to an internal/test context. No recurrence data was available to escalate further." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:27.830583Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/context-overflow/expected.json b/packages/issue-agent/tests/issue-agent/context-overflow/expected.json new file mode 100644 index 0000000..4d8ca4b --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/context-overflow/expected.json @@ -0,0 +1,8 @@ +{ + "classification": "REAL_INCIDENT", + "root_cause_must_contain": ["context window", "truncation", "hallucinat"], + "suspected_components_must_contain": [], + "suggested_investigation_must_contain": ["retrieve", "token", "context"], + "suggested_tests_must_contain": ["token budget", "grounding"], + "confidence_min": 0.5 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/crash-loop/evaluation.json b/packages/issue-agent/tests/issue-agent/crash-loop/evaluation.json new file mode 100644 index 0000000..405cd96 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/crash-loop/evaluation.json @@ -0,0 +1,39 @@ +{ + "evaluation": { + "summary": "The agent entered a loop of repeated tool call failures (MemoryError) while attempting to process a large CSV file. Despite attempting different strategies (pandas, chunking, standard library), the agent failed to identify that the environment constraints were insufficient for the file size, resulting in 4 consecutive OOM errors.", + "classification": "REAL_INCIDENT", + "recoverability": "RECOVERABLE", + "confidence": 1.0, + "failure_modes": [ + "LOOPING", + "TOOL_CALL_ANOMALY" + ], + "suspected_root_cause": "The agent lacks awareness of environment memory limits and continues to attempt memory-intensive operations on a file that exceeds available system resources.", + "suspected_components": [ + "code_interpreter", + "agent_reasoning_loop" + ], + "reasoning": [ + "The agent performed 4 identical tool calls with the same outcome (MemoryError), indicating a failure to adapt to the environment's hard constraints.", + "The agent did not pivot to an alternative strategy (e.g., requesting a smaller subset of data or using an external database/SQL tool) after the first failure.", + "The execution trace shows a clear pattern of repeated tool call failures without state progression." + ], + "recommendations": [ + "Implement a circuit breaker for tool calls that fail due to MemoryError to prevent infinite loops.", + "Provide the agent with metadata about environment resource limits (e.g., available RAM) so it can adjust its strategy before attempting execution.", + "Add a fallback mechanism to notify the user when a task exceeds the capabilities of the current execution environment." + ], + "urgency": { + "tier": "P2", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The incident has terminated and is contained to an internal execution failure. It requires review to improve agent robustness but does not pose an immediate risk to production systems." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:30.602792Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/crash-loop/expected.json b/packages/issue-agent/tests/issue-agent/crash-loop/expected.json new file mode 100644 index 0000000..2bb2f66 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/crash-loop/expected.json @@ -0,0 +1,8 @@ +{ + "classification": "INSUFFICIENT_EVIDENCE", + "summary_must_contain": ["cannot", "blocked", "missing", "trace"], + "evidence_must_contain": [], + "must_not_suggest_fix": true, + "must_not_identify_root_cause": true, + "confidence_max": 0.6 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/critical-escalation/evaluation.json b/packages/issue-agent/tests/issue-agent/critical-escalation/evaluation.json new file mode 100644 index 0000000..deb7e30 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/critical-escalation/evaluation.json @@ -0,0 +1,39 @@ +{ + "evaluation": { + "summary": "The agent correctly identified a payment timeout and escalated the issue to a human supervisor. While the latency is high, the agent followed a logical recovery path for a bank timeout scenario. This is a recurring pattern (3 occurrences), suggesting a systemic issue with the payment processor's response time rather than an agent failure.", + "classification": "REAL_INCIDENT", + "recoverability": "RECOVERABLE", + "confidence": 1.0, + "failure_modes": [ + "NONE_DETECTED" + ], + "suspected_root_cause": "The payment processor is consistently experiencing high latency or bank timeouts, triggering the agent's escalation logic.", + "suspected_components": [ + "payment_processor", + "agent_logic" + ], + "reasoning": [ + "The agent correctly identified that the transaction remained in a 'pending' state after multiple checks.", + "The agent successfully invoked the escalation tool when the bank timeout occurred.", + "The high latency is a symptom of the external service, not an agent malfunction.", + "The recurrence count (3) indicates this is a known, persistent issue with the payment gateway integration." + ], + "recommendations": [ + "Investigate the payment_processor service for performance bottlenecks or increased bank timeout rates.", + "Review the escalation threshold to determine if the 60s wait time is appropriate for current network conditions.", + "Monitor the 'payments-team' queue to ensure the escalated tickets are being addressed." + ], + "urgency": { + "tier": "P2", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The incident is terminated and the agent successfully mitigated the failure by escalating to a human. It is a recurring issue requiring business-hours investigation rather than immediate paging." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:33.474392Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/critical-escalation/expected.json b/packages/issue-agent/tests/issue-agent/critical-escalation/expected.json new file mode 100644 index 0000000..12855e2 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/critical-escalation/expected.json @@ -0,0 +1,8 @@ +{ + "classification": "INSUFFICIENT_EVIDENCE", + "summary_must_contain": ["cannot", "insufficient", "evidence", "trace"], + "evidence_must_contain": [], + "must_not_suggest_fix": true, + "must_not_identify_root_cause": true, + "confidence_max": 0.6 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/example/evaluation.json b/packages/issue-agent/tests/issue-agent/example/evaluation.json new file mode 100644 index 0000000..36c967e --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/example/evaluation.json @@ -0,0 +1,38 @@ +{ + "evaluation": { + "summary": "The agent encountered a tool timeout while fetching weather data for 10 cities, resulting in partial data. It correctly identified the missing cities in its final response, but the high prompt token count (6050) relative to the task suggests potential context bloat or inefficient prompt construction.", + "classification": "REAL_INCIDENT", + "recoverability": "RECOVERABLE", + "confidence": 1.0, + "failure_modes": [ + "TOOL_CALL_ANOMALY" + ], + "suspected_root_cause": "The weather_api tool call timed out due to the request size or external latency, causing a partial data return.", + "suspected_components": [ + "weather_api", + "orchestrator" + ], + "reasoning": [ + "The weather_api call failed with a timeout after 15s, returning only 8 of 10 requested data points.", + "The agent successfully handled the partial data state and reported the missing cities, preventing a hallucination.", + "The high prompt token count (6050) is disproportionate to the simple task, indicating potential inefficiency in context management." + ], + "recommendations": [ + "Implement batching for the weather_api to avoid timeout thresholds.", + "Review prompt construction to reduce token overhead.", + "Add explicit error handling for partial tool responses to ensure the agent explicitly acknowledges missing data rather than just omitting it." + ], + "urgency": { + "tier": "DEFER", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The execution terminated successfully with accurate reporting of the partial data, and there is no evidence of recurring failures." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:35.838438Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/example/expected.json b/packages/issue-agent/tests/issue-agent/example/expected.json new file mode 100644 index 0000000..d0b713c --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/example/expected.json @@ -0,0 +1,8 @@ +{ + "classification": "REAL_INCIDENT", + "root_cause_must_contain": ["weather_api", "timeout", "partial"], + "suspected_components_must_contain": ["weather_api", "orchestrator"], + "suggested_investigation_must_contain": ["batching", "prompt construction", "partial tool responses"], + "suggested_tests_must_contain": ["tool timeout handling", "retry logic"], + "confidence_min": 0.7 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/false-positive/evaluation.json b/packages/issue-agent/tests/issue-agent/false-positive/evaluation.json new file mode 100644 index 0000000..9754f93 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/false-positive/evaluation.json @@ -0,0 +1,35 @@ +{ + "evaluation": { + "summary": "The agent encountered a transient upstream service error during the initial tool call but successfully recovered on the first retry, completing the task as expected.", + "classification": "FALSE_POSITIVE", + "recoverability": "RECOVERABLE", + "confidence": 0.9, + "failure_modes": [ + "NONE_DETECTED" + ], + "suspected_root_cause": "Transient network or upstream service instability causing a temporary failure of the search_api tool.", + "suspected_components": [ + "search_api" + ], + "reasoning": [ + "The agent correctly identified the error in Step 1.", + "The agent successfully implemented a retry mechanism in Step 2.", + "The final output accurately reflects the data retrieved after the successful retry." + ], + "recommendations": [ + "No action required as the agent demonstrated robust error handling and self-recovery." + ], + "urgency": { + "tier": "DEFER", + "page_now": false, + "status": "TERMINATED", + "reasoning": "This was a transient, self-resolved error with no negative impact on the final output." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:37.846094Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/false-positive/expected.json b/packages/issue-agent/tests/issue-agent/false-positive/expected.json new file mode 100644 index 0000000..12855e2 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/false-positive/expected.json @@ -0,0 +1,8 @@ +{ + "classification": "INSUFFICIENT_EVIDENCE", + "summary_must_contain": ["cannot", "insufficient", "evidence", "trace"], + "evidence_must_contain": [], + "must_not_suggest_fix": true, + "must_not_identify_root_cause": true, + "confidence_max": 0.6 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/handoff-failure/evaluation.json b/packages/issue-agent/tests/issue-agent/handoff-failure/evaluation.json new file mode 100644 index 0000000..de13d7a --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/handoff-failure/evaluation.json @@ -0,0 +1,41 @@ +{ + "evaluation": { + "summary": "The researcher agent ignored the planner's explicit constraints (2025) and fetched 2024 data instead, leading to a hallucinated final answer that claimed to address the 2025 request while providing 2024 results.", + "classification": "REAL_INCIDENT", + "recoverability": "RECOVERABLE", + "confidence": 1.0, + "failure_modes": [ + "HANDOFF_FAILURE", + "HALLUCINATION" + ], + "suspected_root_cause": "The researcher agent failed to maintain context consistency from the planner, specifically ignoring the 'year' parameter constraint during the handoff.", + "suspected_components": [ + "planner", + "researcher", + "state_manager" + ], + "reasoning": [ + "The planner explicitly set the constraint to 2025.", + "The researcher agent explicitly stated it would fetch 2024 data instead, violating the established plan.", + "The final output claims to answer the user's 2025 request while providing 2024 data, constituting a hallucination.", + "The trace metadata indicates this is the second occurrence of this failure mode, increasing the severity." + ], + "recommendations": [ + "Implement strict constraint enforcement in the researcher agent to prevent overriding plan parameters.", + "Add a validation step between agents to verify that tool inputs match the original plan constraints.", + "Update the researcher agent's system prompt to prioritize adherence to upstream plan constraints." + ], + "urgency": { + "tier": "P2", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The incident is terminated and the output is contained to an internal report, but the recurrence of this handoff failure warrants review during business hours." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:41.219089Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/handoff-failure/expected.json b/packages/issue-agent/tests/issue-agent/handoff-failure/expected.json new file mode 100644 index 0000000..214d2f4 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/handoff-failure/expected.json @@ -0,0 +1,9 @@ +{ + "classification": "REAL_INCIDENT", + "root_cause_must_contain": ["handoff", "constraints", "context"], + "suspected_components_must_contain": ["researcher_agent", "context_manager"], + "suggested_investigation_must_contain": ["schema", "validation", "contract"], + "suggested_tests_must_contain": ["handoff"], + "relevant_files_must_contain": [], + "confidence_min": 0.5 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/insufficient-evidence/evaluation.json b/packages/issue-agent/tests/issue-agent/insufficient-evidence/evaluation.json new file mode 100644 index 0000000..303f901 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/insufficient-evidence/evaluation.json @@ -0,0 +1,35 @@ +{ + "evaluation": { + "summary": "The incident report provided contains no trace data, making it impossible to perform a forensic analysis of the agent's execution.", + "classification": "INSUFFICIENT_EVIDENCE", + "recoverability": "UNKNOWN", + "confidence": 0.5, + "failure_modes": [ + "NONE_DETECTED" + ], + "suspected_root_cause": "Missing execution trace data for the provided execution ID.", + "suspected_components": [ + "trace_collector" + ], + "reasoning": [ + "The execution trace provided is empty.", + "Without logs, tool calls, or reasoning steps, no failure mode can be identified or verified." + ], + "recommendations": [ + "Verify that the logging infrastructure is correctly capturing traces for execution exec-ie-001.", + "Re-submit the incident with the associated trace data." + ], + "urgency": { + "tier": "DEFER", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The incident is classified as INSUFFICIENT_EVIDENCE due to a lack of trace data, requiring no immediate action." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:43.387883Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/insufficient-evidence/expected.json b/packages/issue-agent/tests/issue-agent/insufficient-evidence/expected.json new file mode 100644 index 0000000..8fa7765 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/insufficient-evidence/expected.json @@ -0,0 +1,8 @@ +{ + "classification": "INSUFFICIENT_EVIDENCE", + "summary_must_contain": ["cannot", "insufficient", "trace", "evidence"], + "evidence_must_contain": [], + "must_not_suggest_fix": true, + "must_not_identify_root_cause": true, + "confidence_max": 0.6 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/looping/evaluation.json b/packages/issue-agent/tests/issue-agent/looping/evaluation.json new file mode 100644 index 0000000..b9b7551 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/looping/evaluation.json @@ -0,0 +1,40 @@ +{ + "evaluation": { + "summary": "The agent entered repeated retries, calling the search_api with identical parameters 7 times despite receiving consistent 'rate limit exceeded' errors, until an internal retry limit terminated the execution.", + "classification": "REAL_INCIDENT", + "recoverability": "RECOVERABLE", + "confidence": 1.0, + "failure_modes": [ + "LOOPING", + "TOOL_CALL_ANOMALY" + ], + "suspected_root_cause": "The agent lacks a backoff strategy or a termination condition when encountering persistent tool errors, causing it to blindly retry the same request until an internal limit is reached.", + "suspected_components": [ + "agent_reasoning_engine", + "search_api_client" + ], + "reasoning": [ + "The agent performed 7 identical tool calls with the same input.", + "The agent ignored the 'rate limit exceeded' error message which explicitly requested a retry delay.", + "There was no state change or adaptive behavior observed across the 7 steps.", + "The execution terminated only after reaching an internal retry limit, not because the task was completed." + ], + "recommendations": [ + "Implement an exponential backoff strategy for tool calls.", + "Add a circuit breaker pattern to prevent the agent from retrying after a threshold of consecutive failures.", + "Update the agent's system prompt to handle specific error codes (like 429) by waiting or escalating rather than immediate retries." + ], + "urgency": { + "tier": "P2", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The incident is terminated and contained to an internal execution failure; it does not require immediate paging but should be addressed to prevent resource waste." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:46.181314Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/looping/expected.json b/packages/issue-agent/tests/issue-agent/looping/expected.json new file mode 100644 index 0000000..9ecf48e --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/looping/expected.json @@ -0,0 +1,9 @@ +{ + "classification": "REAL_INCIDENT", + "root_cause_must_contain": ["loop", "retry", "backoff"], + "suspected_components_must_contain": ["orchestrat", "search"], + "suggested_investigation_must_contain": ["backoff", "retry", "threshold"], + "suggested_tests_must_contain": [], + "relevant_files_must_contain": [], + "confidence_min": 0.5 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/mixed-labels/evaluation.json b/packages/issue-agent/tests/issue-agent/mixed-labels/evaluation.json new file mode 100644 index 0000000..4e08c15 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/mixed-labels/evaluation.json @@ -0,0 +1,38 @@ +{ + "evaluation": { + "summary": "The agent encountered a context window overflow during an initial attempt to generate a comprehensive financial report, which triggered a retry mechanism that successfully completed a partial task.", + "classification": "REAL_INCIDENT", + "recoverability": "RECOVERABLE", + "confidence": 0.9, + "failure_modes": [ + "SILENT_CONTEXT_OVERFLOW" + ], + "suspected_root_cause": "The agent attempted to process an excessively large prompt exceeding the 16K token limit in a single request, necessitating a fallback strategy.", + "suspected_components": [ + "llm_generate", + "prompt_orchestrator" + ], + "reasoning": [ + "Step 1 explicitly returned a 500 error due to context exceeding the 16K limit.", + "The agent successfully recovered by decomposing the task into smaller, manageable sections in Step 2.", + "The incident is classified as a REAL_INCIDENT because the initial attempt failed due to poor prompt management, even though it self-resolved." + ], + "recommendations": [ + "Implement a pre-flight token counter to validate prompt size before invoking the llm_generate tool.", + "Automate the decomposition of large reporting tasks into smaller sub-tasks to prevent context overflow.", + "Update the agent's error handling logic to automatically switch to a chunked generation strategy when context limits are approached." + ], + "urgency": { + "tier": "DEFER", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The failure was transient and self-corrected via a retry mechanism; no downstream damage occurred as the task was successfully completed." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:48.878093Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/mixed-labels/expected.json b/packages/issue-agent/tests/issue-agent/mixed-labels/expected.json new file mode 100644 index 0000000..1effadc --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/mixed-labels/expected.json @@ -0,0 +1,8 @@ +{ + "classification": "REAL_INCIDENT", + "root_cause_must_contain": ["context", "overflow", "16K", "token"], + "suspected_components_must_contain": ["llm_generate", "prompt_orchestrator"], + "suggested_investigation_must_contain": ["token counter", "decomposition", "chunked generation"], + "suggested_tests_must_contain": ["token budget", "prompt size validation"], + "confidence_min": 0.7 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/rate-limit/evaluation.json b/packages/issue-agent/tests/issue-agent/rate-limit/evaluation.json new file mode 100644 index 0000000..50ecbd8 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/rate-limit/evaluation.json @@ -0,0 +1,37 @@ +{ + "evaluation": { + "summary": "The agent encountered a transient HTTP 429 rate limit error during an external API call. It successfully implemented a retry mechanism, retrieved the necessary data, and completed the task without further issues.", + "classification": "FALSE_POSITIVE", + "recoverability": "RECOVERABLE", + "confidence": 0.9, + "failure_modes": [ + "NONE_DETECTED" + ], + "suspected_root_cause": "Transient rate limiting from the external supplier API, which was successfully handled by the agent's retry logic.", + "suspected_components": [ + "external_api_client" + ], + "reasoning": [ + "The agent correctly identified the 429 error code.", + "The agent implemented a retry strategy and successfully obtained the required data on the second attempt.", + "The final output accurately reflects the retrieved inventory data.", + "The incident is a standard operational recovery rather than a failure." + ], + "recommendations": [ + "Monitor the frequency of 429 errors to determine if the current retry backoff strategy needs adjustment.", + "No immediate code changes required as the agent handled the error gracefully." + ], + "urgency": { + "tier": "DEFER", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The incident was a self-resolved transient error that did not impact the final output or system stability." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:51.150416Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/rate-limit/expected.json b/packages/issue-agent/tests/issue-agent/rate-limit/expected.json new file mode 100644 index 0000000..60c2fac --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/rate-limit/expected.json @@ -0,0 +1,8 @@ +{ + "classification": "INSUFFICIENT_EVIDENCE", + "summary_must_contain": ["cannot", "missing", "trace", "evidence"], + "evidence_must_contain": [], + "must_not_suggest_fix": true, + "must_not_identify_root_cause": true, + "confidence_max": 0.6 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/recurring/evaluation.json b/packages/issue-agent/tests/issue-agent/recurring/evaluation.json new file mode 100644 index 0000000..e2d9245 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/recurring/evaluation.json @@ -0,0 +1,39 @@ +{ + "evaluation": { + "summary": "The agent successfully completed its task of fetching sales data, but exhibited high latency across multiple tool calls. Given the high occurrence count (12) and consistent latency patterns, this indicates a performance bottleneck in the data_fetch tool rather than an agent logic failure.", + "classification": "REAL_INCIDENT", + "recoverability": "RECOVERABLE", + "confidence": 1.0, + "failure_modes": [ + "TOOL_CALL_ANOMALY" + ], + "suspected_root_cause": "The 'data_fetch' tool is experiencing significant latency, likely due to inefficient database queries or downstream service congestion when processing 'daily_sales' and 'sales_by_region' datasets.", + "suspected_components": [ + "data_fetch_tool", + "database_layer" + ], + "reasoning": [ + "Total execution latency reached 15.7 seconds for only two tool calls.", + "Individual tool calls averaged over 7 seconds, which is significantly higher than expected for standard data retrieval.", + "The incident has occurred 12 times, indicating a persistent performance degradation rather than a transient network blip.", + "The agent logic itself functioned correctly and produced the expected output, isolating the issue to the tool performance." + ], + "recommendations": [ + "Investigate the 'data_fetch' tool implementation for inefficient query patterns or missing indexes.", + "Implement caching for 'daily_sales' and 'sales_by_region' datasets to reduce redundant tool calls.", + "Set up performance monitoring alerts for the 'data_fetch' tool to trigger when latency exceeds 2000ms." + ], + "urgency": { + "tier": "P2", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The execution has terminated and the output is correct, but the high recurrence rate of latency issues warrants investigation during business hours to prevent user-facing timeouts." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:53.767295Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/recurring/expected.json b/packages/issue-agent/tests/issue-agent/recurring/expected.json new file mode 100644 index 0000000..edf736e --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/recurring/expected.json @@ -0,0 +1,8 @@ +{ + "classification": "REAL_INCIDENT", + "root_cause_must_contain": ["data_fetch", "latency", "database", "query"], + "suspected_components_must_contain": ["data_fetch_tool", "database_layer"], + "suggested_investigation_must_contain": ["query patterns", "indexes", "caching", "monitoring"], + "suggested_tests_must_contain": ["latency threshold", "performance regression"], + "confidence_min": 0.8 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/silent-hallucination/evaluation.json b/packages/issue-agent/tests/issue-agent/silent-hallucination/evaluation.json new file mode 100644 index 0000000..012db56 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/silent-hallucination/evaluation.json @@ -0,0 +1,39 @@ +{ + "evaluation": { + "summary": "The agent performed a knowledge base search that returned $2.1B, but subsequently ignored this retrieved context to provide a hallucinated figure of $3.8B in its final response.", + "classification": "REAL_INCIDENT", + "recoverability": "RECOVERABLE", + "confidence": 1.0, + "failure_modes": [ + "HALLUCINATION", + "STALE_CONTEXT" + ], + "suspected_root_cause": "The model prioritized its internal pre-trained knowledge over the provided tool output, failing to ground its final response in the retrieved context.", + "suspected_components": [ + "LLM_Inference_Engine", + "Context_Manager" + ], + "reasoning": [ + "The tool output explicitly stated revenue was $2.1B.", + "The agent's final response claimed revenue was $3.8B, explicitly citing its own knowledge rather than the retrieved document.", + "The agent ignored the retrieved evidence despite the tool call succeeding without error." + ], + "recommendations": [ + "Implement a system prompt constraint requiring the agent to cite specific tool outputs for factual claims.", + "Introduce a post-generation validation step to verify final answers against retrieved context.", + "Adjust temperature settings to reduce creative generation when factual accuracy is required." + ], + "urgency": { + "tier": "P2", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The execution has terminated and the output is incorrect, but it is an internal-facing task that is easily re-runnable and does not pose an immediate external risk." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:56.553224Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/silent-hallucination/expected.json b/packages/issue-agent/tests/issue-agent/silent-hallucination/expected.json new file mode 100644 index 0000000..28c5b87 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/silent-hallucination/expected.json @@ -0,0 +1,9 @@ +{ + "classification": "REAL_INCIDENT", + "root_cause_must_contain": ["hallucinat", "context", "ground"], + "suspected_components_must_contain": ["inference", "LLM", "Context"], + "suggested_investigation_must_contain": ["grounding", "validation", "verification"], + "suggested_tests_must_contain": [], + "relevant_files_must_contain": [], + "confidence_min": 0.5 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/tool-anomaly/evaluation.json b/packages/issue-agent/tests/issue-agent/tool-anomaly/evaluation.json new file mode 100644 index 0000000..3aee2cf --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/tool-anomaly/evaluation.json @@ -0,0 +1,40 @@ +{ + "evaluation": { + "summary": "The agent hallucinated the success of an email delivery after a tool-reported validation failure and a subsequent delivery error.", + "classification": "REAL_INCIDENT", + "recoverability": "RECOVERABLE", + "confidence": 1.0, + "failure_modes": [ + "HALLUCINATION", + "TOOL_CALL_ANOMALY" + ], + "suspected_root_cause": "The agent ignored the tool output indicating an invalid email address and the subsequent error message from the send_email tool, leading to a false positive completion state.", + "suspected_components": [ + "reasoning_engine", + "tool_execution_handler" + ], + "reasoning": [ + "Step 1 explicitly returned 'valid: false' for the email address.", + "Step 2 returned a delivery failure error from the send_email tool.", + "Step 3 falsely claimed 'The email was sent successfully' despite clear evidence of failure in the trace history.", + "The agent failed to implement error handling logic to stop the workflow upon receiving a negative tool output." + ], + "recommendations": [ + "Implement a mandatory check for tool output status before proceeding to subsequent steps.", + "Add a guardrail to prevent the agent from finalizing a task if a critical tool call returns an error status.", + "Update the system prompt to explicitly instruct the agent to halt and report errors if validation tools return false." + ], + "urgency": { + "tier": "P2", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The execution has terminated and the impact is contained to a single failed email notification, which is re-runnable and does not pose an immediate ongoing threat." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:14:59.084281Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/tool-anomaly/expected.json b/packages/issue-agent/tests/issue-agent/tool-anomaly/expected.json new file mode 100644 index 0000000..9cf58ea --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/tool-anomaly/expected.json @@ -0,0 +1,9 @@ +{ + "classification": "REAL_INCIDENT", + "root_cause_must_contain": ["validat", "tool", "fail"], + "suspected_components_must_contain": ["orchestrat", "loop"], + "suggested_investigation_must_contain": ["error", "tool"], + "suggested_tests_must_contain": [], + "relevant_files_must_contain": [], + "confidence_min": 0.5 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/transient-error/evaluation.json b/packages/issue-agent/tests/issue-agent/transient-error/evaluation.json new file mode 100644 index 0000000..18718a3 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/transient-error/evaluation.json @@ -0,0 +1,37 @@ +{ + "evaluation": { + "summary": "The agent encountered a transient 429 rate limit error on its first attempt to call the exchange_rates tool. It successfully implemented a retry mechanism, retrieved the required data, and completed the task as expected.", + "classification": "FALSE_POSITIVE", + "recoverability": "RECOVERABLE", + "confidence": 0.9, + "failure_modes": [ + "NONE_DETECTED" + ], + "suspected_root_cause": "Transient API rate limiting which was successfully handled by the agent's retry logic.", + "suspected_components": [ + "rate_limited_api" + ], + "reasoning": [ + "The agent correctly identified the rate limit error in Step 1.", + "The agent successfully performed a retry in Step 2, which resulted in a successful tool execution.", + "The final output in Step 3 accurately reflects the data retrieved from the successful tool call.", + "The incident is a standard transient error handled by built-in retry mechanisms." + ], + "recommendations": [ + "No action required as the agent demonstrated successful self-recovery.", + "Monitor the rate_limited_api for increased frequency of 429 errors to determine if the current retry strategy remains sufficient." + ], + "urgency": { + "tier": "DEFER", + "page_now": false, + "status": "TERMINATED", + "reasoning": "The execution completed successfully after a single retry, indicating the system is functioning as designed." + } + }, + "metadata": { + "prompt_version": "3.0.0", + "model_version": "gemini-3.1-flash-lite", + "model_temperature": 0.2, + "evaluated_at": "2026-07-23T13:15:01.441142Z" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/issue-agent/transient-error/expected.json b/packages/issue-agent/tests/issue-agent/transient-error/expected.json new file mode 100644 index 0000000..43e9563 --- /dev/null +++ b/packages/issue-agent/tests/issue-agent/transient-error/expected.json @@ -0,0 +1,8 @@ +{ + "classification": "INSUFFICIENT_EVIDENCE", + "summary_must_contain": ["cannot", "missing", "trace", "logging"], + "evidence_must_contain": [], + "must_not_suggest_fix": true, + "must_not_identify_root_cause": true, + "confidence_max": 0.6 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/run_all.sh b/packages/issue-agent/tests/run_all.sh new file mode 100755 index 0000000..e7a992b --- /dev/null +++ b/packages/issue-agent/tests/run_all.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../../.." && pwd)" +cd "$ROOT" + +if [ -f .env ]; then + set -a; source .env; set +a +fi + +export PYTHONPATH="$ROOT/packages/evaluator/src:$ROOT/packages/issue-agent/src:$HERE" +VENV="$ROOT/packages/evaluator/.venv/bin/python3" + +echo "=== Unit Tests ===" +$VENV -m unittest \ + "$HERE/test_schemas.py" \ + "$HERE/test_agent.py" \ + "$HERE/test_repository.py" \ + "$HERE/test_mapper.py" \ + -v \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/context_overflow/evaluator.json b/packages/issue-agent/tests/scenarios/context_overflow/evaluator.json new file mode 100644 index 0000000..26fafb2 --- /dev/null +++ b/packages/issue-agent/tests/scenarios/context_overflow/evaluator.json @@ -0,0 +1,7 @@ +{ + "failure_modes": ["SILENT_CONTEXT_OVERFLOW"], + "confidence": 0.9, + "reasoning": "Agent exceeded context window by loading entire document without truncation.", + "urgency_tier": "P1", + "severity": "HIGH" +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/context_overflow/expected.json b/packages/issue-agent/tests/scenarios/context_overflow/expected.json new file mode 100644 index 0000000..a10ce46 --- /dev/null +++ b/packages/issue-agent/tests/scenarios/context_overflow/expected.json @@ -0,0 +1,6 @@ +{ + "suspected_component": "context_manager", + "suspected_files": ["context_manager.py", "tool_executor.py", "planner.py"], + "root_cause_pattern": "no context window monitoring or truncation", + "confidence_threshold": 0.7 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/context_overflow/incident.json b/packages/issue-agent/tests/scenarios/context_overflow/incident.json new file mode 100644 index 0000000..4118a6d --- /dev/null +++ b/packages/issue-agent/tests/scenarios/context_overflow/incident.json @@ -0,0 +1,66 @@ +{ + "incident_id": "context_overflow_001", + "execution_trace": { + "agent_steps": [ + { + "step_type": "tool_call", + "planner_output": "Continue processing the document", + "tool_calls": [ + { + "name": "read_document", + "input": "doc_id=large_report_v3", + "output": "[12000 tokens of document content]", + "latency_ms": 15000, + "success": true, + "error": null + }, + { + "name": "summarize", + "input": "Summarize the document", + "output": null, + "latency_ms": 30000, + "success": false, + "error": "Context length exceeded maximum of 8192 tokens" + } + ], + "context": "Processing annual report. Document size exceeds available context window.", + "latency_ms": 50000 + }, + { + "step_type": "tool_call", + "planner_output": "Retry with truncation", + "tool_calls": [ + { + "name": "summarize", + "input": "Summarize the document", + "output": null, + "latency_ms": 30000, + "success": false, + "error": "Context length exceeded maximum of 8192 tokens" + } + ], + "context": "Planner retried with same full document instead of truncating", + "latency_ms": 35000 + } + ], + "model": "gpt-4o", + "total_latency_ms": 85000, + "tokens_used": 24000 + }, + "evaluation": { + "failure_modes": ["SILENT_CONTEXT_OVERFLOW"], + "confidence": 0.9, + "reasoning": "Agent exceeded context window by loading entire document without truncation. The planner retried the same operation instead of adjusting the input size. No context window monitoring or truncation strategy exists.", + "urgency_tier": "P1", + "severity": "HIGH" + }, + "telemetry": { + "context_utilization_pct": 146, + "max_context_tokens": 8192, + "document_size_tokens": 12000 + }, + "metadata": { + "customer": "data_analytics", + "environment": "production" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/handoff_failure/evaluator.json b/packages/issue-agent/tests/scenarios/handoff_failure/evaluator.json new file mode 100644 index 0000000..f0007bc --- /dev/null +++ b/packages/issue-agent/tests/scenarios/handoff_failure/evaluator.json @@ -0,0 +1,7 @@ +{ + "failure_modes": ["HANDOFF_CONTEXT_LOSS"], + "confidence": 0.95, + "reasoning": "Cart state was lost during agent handoff.", + "urgency_tier": "P1", + "severity": "CRITICAL" +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/handoff_failure/expected.json b/packages/issue-agent/tests/scenarios/handoff_failure/expected.json new file mode 100644 index 0000000..2ff4b6f --- /dev/null +++ b/packages/issue-agent/tests/scenarios/handoff_failure/expected.json @@ -0,0 +1,6 @@ +{ + "suspected_component": "handoff_serializer", + "suspected_files": ["handoff.py", "state_manager.py"], + "root_cause_pattern": "state not serialized during handoff", + "confidence_threshold": 0.7 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/handoff_failure/incident.json b/packages/issue-agent/tests/scenarios/handoff_failure/incident.json new file mode 100644 index 0000000..463733b --- /dev/null +++ b/packages/issue-agent/tests/scenarios/handoff_failure/incident.json @@ -0,0 +1,49 @@ +{ + "incident_id": "handoff_failure_001", + "execution_trace": { + "agent_steps": [ + { + "step_type": "handoff", + "planner_output": "Handling off to payment_agent for checkout processing", + "tool_calls": [], + "context": "Cart total: $299.99. Handoff to payment_agent.", + "latency_ms": 5000 + }, + { + "step_type": "handoff", + "planner_output": "Payment agent received empty cart state", + "tool_calls": [ + { + "name": "process_payment", + "input": "amount=0", + "output": "{\"status\": \"failed\", \"reason\": \"invalid_amount\"}", + "latency_ms": 100, + "success": false, + "error": "Payment declined: amount must be greater than 0" + } + ], + "context": "Cart state was not serialized during handoff", + "latency_ms": 6000 + } + ], + "model": "gpt-4o", + "total_latency_ms": 11000, + "tokens_used": 2200 + }, + "evaluation": { + "failure_modes": ["HANDOFF_CONTEXT_LOSS"], + "confidence": 0.95, + "reasoning": "Cart state was lost during agent handoff. Payment agent received empty state. Handoff serializer does not include conversation context or intermediate state.", + "urgency_tier": "P1", + "severity": "CRITICAL" + }, + "telemetry": { + "handoff_count": 2, + "state_serialization_errors": 1, + "context_size_bytes": 0 + }, + "metadata": { + "customer": "enterprise_client", + "environment": "staging" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/planner_bug/evaluator.json b/packages/issue-agent/tests/scenarios/planner_bug/evaluator.json new file mode 100644 index 0000000..859747e --- /dev/null +++ b/packages/issue-agent/tests/scenarios/planner_bug/evaluator.json @@ -0,0 +1,7 @@ +{ + "failure_modes": ["INEFFICIENT_PLANNING"], + "confidence": 0.88, + "reasoning": "Planner called get_weather sequentially for 47 cities instead of batching.", + "urgency_tier": "P2", + "severity": "MEDIUM" +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/planner_bug/expected.json b/packages/issue-agent/tests/scenarios/planner_bug/expected.json new file mode 100644 index 0000000..c2a5cf3 --- /dev/null +++ b/packages/issue-agent/tests/scenarios/planner_bug/expected.json @@ -0,0 +1,6 @@ +{ + "suspected_component": "planner", + "suspected_files": ["planner.py", "tool_registry.py"], + "root_cause_pattern": "sequential tool execution without batching", + "confidence_threshold": 0.7 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/planner_bug/incident.json b/packages/issue-agent/tests/scenarios/planner_bug/incident.json new file mode 100644 index 0000000..1876b5f --- /dev/null +++ b/packages/issue-agent/tests/scenarios/planner_bug/incident.json @@ -0,0 +1,81 @@ +{ + "incident_id": "planner_bug_001", + "execution_trace": { + "agent_steps": [ + { + "step_type": "plan", + "planner_output": "I need to check the weather, so I'll call get_weather for each city in sequence", + "tool_calls": [ + {"name": "get_weather", "input": "city=New York", "output": "{\"temp\": 22}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=London", "output": "{\"temp\": 15}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Tokyo", "output": "{\"temp\": 28}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Paris", "output": "{\"temp\": 18}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Sydney", "output": "{\"temp\": 12}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Berlin", "output": "{\"temp\": 20}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Moscow", "output": "{\"temp\": -5}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Beijing", "output": "{\"temp\": 25}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Delhi", "output": "{\"temp\": 32}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Shanghai", "output": "{\"temp\": 27}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Sao Paulo", "output": "{\"temp\": 24}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Mexico City", "output": "{\"temp\": 21}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Cairo", "output": "{\"temp\": 30}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Lagos", "output": "{\"temp\": 28}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Istanbul", "output": "{\"temp\": 19}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Los Angeles", "output": "{\"temp\": 23}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Chicago", "output": "{\"temp\": 10}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Toronto", "output": "{\"temp\": 5}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Houston", "output": "{\"temp\": 26}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Miami", "output": "{\"temp\": 29}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Dallas", "output": "{\"temp\": 24}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=San Francisco", "output": "{\"temp\": 17}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Seattle", "output": "{\"temp\": 14}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Boston", "output": "{\"temp\": 8}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Atlanta", "output": "{\"temp\": 22}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Phoenix", "output": "{\"temp\": 35}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Denver", "output": "{\"temp\": 12}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Detroit", "output": "{\"temp\": 6}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Minneapolis", "output": "{\"temp\": -2}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=San Diego", "output": "{\"temp\": 21}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Orlando", "output": "{\"temp\": 27}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Tampa", "output": "{\"temp\": 28}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Charlotte", "output": "{\"temp\": 19}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Raleigh", "output": "{\"temp\": 18}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Nashville", "output": "{\"temp\": 16}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Portland", "output": "{\"temp\": 13}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Salt Lake City", "output": "{\"temp\": 11}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Las Vegas", "output": "{\"temp\": 33}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Sacramento", "output": "{\"temp\": 20}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Kansas City", "output": "{\"temp\": 14}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Cincinnati", "output": "{\"temp\": 12}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Milwaukee", "output": "{\"temp\": 9}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Columbus", "output": "{\"temp\": 15}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Indianapolis", "output": "{\"temp\": 11}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=San Antonio", "output": "{\"temp\": 25}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Providence", "output": "{\"temp\": 7}", "latency_ms": 200, "success": true, "error": null}, + {"name": "get_weather", "input": "city=Richmond", "output": "{\"temp\": 17}", "latency_ms": 200, "success": true, "error": null} + ], + "context": "User asked: compare weather across 50 cities", + "latency_ms": 30000 + } + ], + "model": "gpt-4o", + "total_latency_ms": 30000, + "tokens_used": 8500 + }, + "evaluation": { + "failure_modes": ["INEFFICIENT_PLANNING"], + "confidence": 0.88, + "reasoning": "Planner called get_weather sequentially for 47 cities instead of batching. Each call added 200ms latency. Total 9.4s vs ~200ms if batched. Planner lacks awareness of tool batching capabilities.", + "urgency_tier": "P2", + "severity": "MEDIUM" + }, + "telemetry": { + "avg_tool_latency_ms": 200, + "total_tool_calls": 47, + "parallelism": 1 + }, + "metadata": { + "customer": "beta_user", + "environment": "production" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/tool_call_anomaly/evaluator.json b/packages/issue-agent/tests/scenarios/tool_call_anomaly/evaluator.json new file mode 100644 index 0000000..db00e35 --- /dev/null +++ b/packages/issue-agent/tests/scenarios/tool_call_anomaly/evaluator.json @@ -0,0 +1,7 @@ +{ + "failure_modes": ["PARTIAL_TOOL_SUCCESS"], + "confidence": 0.85, + "reasoning": "get_user_profile returned null preferences but the planner did not validate the response before calling recommend_products.", + "urgency_tier": "P2", + "severity": "MEDIUM" +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/tool_call_anomaly/expected.json b/packages/issue-agent/tests/scenarios/tool_call_anomaly/expected.json new file mode 100644 index 0000000..ab4efda --- /dev/null +++ b/packages/issue-agent/tests/scenarios/tool_call_anomaly/expected.json @@ -0,0 +1,6 @@ +{ + "suspected_component": "tool_response_validator", + "suspected_files": ["tool_executor.py", "response_validator.py"], + "root_cause_pattern": "null field not validated before downstream use", + "confidence_threshold": 0.7 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/tool_call_anomaly/incident.json b/packages/issue-agent/tests/scenarios/tool_call_anomaly/incident.json new file mode 100644 index 0000000..9ae54e7 --- /dev/null +++ b/packages/issue-agent/tests/scenarios/tool_call_anomaly/incident.json @@ -0,0 +1,58 @@ +{ + "incident_id": "tool_anomaly_001", + "execution_trace": { + "agent_steps": [ + { + "step_type": "tool_call", + "planner_output": "Fetch user data for personalization", + "tool_calls": [ + { + "name": "get_user_profile", + "input": "user_id=abc123", + "output": "{\"name\": \"John\", \"preferences\": null, \"role\": \"admin\"}", + "latency_ms": 300, + "success": true, + "error": null + }, + { + "name": "get_user_permissions", + "input": "user_id=abc123", + "output": "null", + "latency_ms": 50, + "success": true, + "error": null + }, + { + "name": "recommend_products", + "input": "user_id=abc123, preferences=null", + "output": "{\"error\": \"cannot recommend without preferences\"}", + "latency_ms": 100, + "success": false, + "error": "recommend_products failed: preferences is null" + } + ], + "context": "User has no preferences set. Tool returned null for preferences but continued execution.", + "latency_ms": 1000 + } + ], + "model": "gpt-4o", + "total_latency_ms": 1000, + "tokens_used": 800 + }, + "evaluation": { + "failure_modes": ["PARTIAL_TOOL_SUCCESS"], + "confidence": 0.85, + "reasoning": "get_user_profile returned null preferences but the planner did not validate the response before calling recommend_products. Silent data quality issue propagated through the pipeline.", + "urgency_tier": "P2", + "severity": "MEDIUM" + }, + "telemetry": { + "null_field_count": 1, + "propagation_depth": 2, + "error_chain_length": 1 + }, + "metadata": { + "customer": "consumer_app", + "environment": "production" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/tool_timeout/evaluator.json b/packages/issue-agent/tests/scenarios/tool_timeout/evaluator.json new file mode 100644 index 0000000..45e73db --- /dev/null +++ b/packages/issue-agent/tests/scenarios/tool_timeout/evaluator.json @@ -0,0 +1,7 @@ +{ + "failure_modes": ["TOOL_TIMEOUT"], + "confidence": 0.92, + "reasoning": "Tool search_documents timed out twice consecutively with no fallback. The tool executor lacks timeout configuration and retry logic.", + "urgency_tier": "P2", + "severity": "HIGH" +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/tool_timeout/expected.json b/packages/issue-agent/tests/scenarios/tool_timeout/expected.json new file mode 100644 index 0000000..cf41874 --- /dev/null +++ b/packages/issue-agent/tests/scenarios/tool_timeout/expected.json @@ -0,0 +1,6 @@ +{ + "suspected_component": "tool_executor", + "suspected_files": ["tool_executor.py", "retry.py"], + "root_cause_pattern": "missing timeout configuration", + "confidence_threshold": 0.7 +} \ No newline at end of file diff --git a/packages/issue-agent/tests/scenarios/tool_timeout/incident.json b/packages/issue-agent/tests/scenarios/tool_timeout/incident.json new file mode 100644 index 0000000..28ea819 --- /dev/null +++ b/packages/issue-agent/tests/scenarios/tool_timeout/incident.json @@ -0,0 +1,50 @@ +{ + "incident_id": "tool_timeout_001", + "execution_trace": { + "agent_steps": [ + { + "step_type": "tool_call", + "planner_output": "Search for relevant documents", + "tool_calls": [ + { + "name": "search_documents", + "input": "query=latest_pricing", + "output": null, + "latency_ms": 60000, + "success": false, + "error": "TIMEOUT after 60000ms" + }, + { + "name": "search_documents", + "input": "query=latest_pricing", + "output": null, + "latency_ms": 60000, + "success": false, + "error": "TIMEOUT after 60000ms" + } + ], + "context": "User asked for latest pricing data", + "latency_ms": 125000 + } + ], + "model": "gpt-4o", + "total_latency_ms": 125000, + "tokens_used": 1500 + }, + "evaluation": { + "failure_modes": ["TOOL_TIMEOUT"], + "confidence": 0.92, + "reasoning": "Tool search_documents timed out twice consecutively with no fallback. The tool executor lacks timeout configuration and retry logic.", + "urgency_tier": "P2", + "severity": "HIGH" + }, + "telemetry": { + "tool_latency_p99_ms": 60000, + "tool_error_rate": 0.5, + "concurrent_requests": 12 + }, + "metadata": { + "customer": "acme_corp", + "environment": "production" + } +} \ No newline at end of file diff --git a/packages/issue-agent/tests/test_agent.py b/packages/issue-agent/tests/test_agent.py new file mode 100644 index 0000000..627aadc --- /dev/null +++ b/packages/issue-agent/tests/test_agent.py @@ -0,0 +1,65 @@ +import unittest +import json +import os +from issue_agent.schemas import IncidentSnapshot +from issue_agent.evidence import extract_evidence + + +class TestExtractEvidence(unittest.TestCase): + def setUp(self): + fixture_path = os.path.join(os.path.dirname(__file__), "scenarios", "tool_timeout", "incident.json") + with open(fixture_path) as f: + data = json.load(f) + self.snapshot = IncidentSnapshot.model_validate(data) + + def test_extract_evidence_from_tool_timeout(self): + evidence = extract_evidence(self.snapshot) + self.assertGreater(len(evidence), 0) + self.assertIn("TOOL_TIMEOUT", [e.failure_mode for e in evidence]) + for e in evidence: + self.assertGreaterEqual(e.confidence, 0.0) + self.assertLessEqual(e.confidence, 1.0) + + def test_extract_evidence_includes_failed_tool_calls(self): + evidence = extract_evidence(self.snapshot) + for e in evidence: + if e.failure_mode == "TOOL_TIMEOUT": + self.assertTrue( + any("search_documents" in line for line in e.supporting_trace), + ) + + +class TestEvidenceAcrossScenarios(unittest.TestCase): + def _load_scenario(self, name): + fixture_path = os.path.join(os.path.dirname(__file__), "scenarios", name, "incident.json") + with open(fixture_path) as f: + data = json.load(f) + return IncidentSnapshot.model_validate(data) + + def test_planner_bug_evidence(self): + snapshot = self._load_scenario("planner_bug") + evidence = extract_evidence(snapshot) + self.assertGreater(len(evidence), 0) + self.assertIn("INEFFICIENT_PLANNING", [e.failure_mode for e in evidence]) + + def test_handoff_failure_evidence(self): + snapshot = self._load_scenario("handoff_failure") + evidence = extract_evidence(snapshot) + self.assertGreater(len(evidence), 0) + self.assertIn("HANDOFF_CONTEXT_LOSS", [e.failure_mode for e in evidence]) + + def test_context_overflow_evidence(self): + snapshot = self._load_scenario("context_overflow") + evidence = extract_evidence(snapshot) + self.assertGreater(len(evidence), 0) + self.assertIn("SILENT_CONTEXT_OVERFLOW", [e.failure_mode for e in evidence]) + + def test_tool_anomaly_evidence(self): + snapshot = self._load_scenario("tool_call_anomaly") + evidence = extract_evidence(snapshot) + self.assertGreater(len(evidence), 0) + self.assertIn("PARTIAL_TOOL_SUCCESS", [e.failure_mode for e in evidence]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/packages/issue-agent/tests/test_helpers.py b/packages/issue-agent/tests/test_helpers.py new file mode 100644 index 0000000..764028b --- /dev/null +++ b/packages/issue-agent/tests/test_helpers.py @@ -0,0 +1,258 @@ +import json +import logging +import os +import sys + + +_HERE = os.path.dirname(__file__) +_PROJECT_ROOT = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) + +_log = logging.getLogger("issue_agent") +_log.setLevel(logging.DEBUG) +_handler = logging.StreamHandler(sys.stderr) +_handler.setFormatter(logging.Formatter("\n[issue-agent] %(levelname)s %(message)s\n")) +_log.addHandler(_handler) +_log.propagate = False + +_captured_logs: list[dict] = [] + + +class _LogCapture(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + d = {"level": record.levelname, "msg": record.getMessage()} + extras = {k: v for k, v in record.__dict__.items() if k not in logging.LogRecord("n", 0, "", 0, "", (), None).__dict__} + if extras: + d["extra"] = extras + _captured_logs.append(d) + + +_capture_handler = _LogCapture() +_capture_handler.setLevel(logging.DEBUG) +_log.addHandler(_capture_handler) + + +_SEVERITY_MAP = {"P0": "CRITICAL", "P1": "HIGH", "P2": "MEDIUM", "DEFER": "LOW"} + +def _extract_severity(eval_data: dict) -> str: + raw = eval_data.get("severity", eval_data.get("urgency_tier", eval_data.get("urgency", "MEDIUM"))) + if isinstance(raw, dict): + raw = raw.get("tier", str(raw.get("status", "MEDIUM"))) + mapped = _SEVERITY_MAP.get(raw) + if mapped: + return mapped + if isinstance(raw, str) and raw in ("CRITICAL", "HIGH", "MEDIUM", "LOW"): + return raw + return "MEDIUM" + + +def _load_dotenv(): + dotenv = os.path.join(_PROJECT_ROOT, ".env") + if not os.path.exists(dotenv): + return + with open(dotenv) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, val = line.split("=", 1) + key = key.strip() + val = val.strip().strip('"').strip("'") + if key and not os.environ.get(key): + os.environ[key] = val + + +_load_dotenv() +EVALUATOR_SRC = os.path.join(_PROJECT_ROOT, "packages", "evaluator", "src") +ISSUE_AGENT_SRC = os.path.join(_PROJECT_ROOT, "packages", "issue-agent", "src") + +if EVALUATOR_SRC not in sys.path: + sys.path.insert(0, EVALUATOR_SRC) +if ISSUE_AGENT_SRC not in sys.path: + sys.path.insert(0, ISSUE_AGENT_SRC) + + +from issue_agent.schemas import IncidentSnapshot, ExecutionTrace, AgentStep, ToolCall, EvaluationResult +from issue_agent.agent import run_issue_agent +from issue_agent.repository import LocalRepo + + +def load_json(path: str) -> dict: + with open(path) as f: + return json.load(f) + + +def run_evaluator(incident: dict) -> dict: + from evaluator.agent import Agent as EvaluatorAgent + agent = EvaluatorAgent() + result = agent.evaluate(incident) + if result is None: + return {"evaluation": {}, "metadata": {}} + return json.loads(result.model_dump_json()) + + +def build_snapshot(incident: dict, evaluator: dict) -> IncidentSnapshot: + eval_data = evaluator.get("evaluation", {}) + failure_modes = eval_data.get("failure_modes", []) + if not failure_modes: + failure_modes = incident.get("latest_labels", ["NONE_DETECTED"]) + + steps = [] + for s in incident.get("agent_steps", []): + llm = s.get("llm_response", {}) + tool_calls = [ + ToolCall( + name=tc.get("name", ""), + input=tc.get("input", ""), + output=tc.get("output"), + latency_ms=tc.get("latency_ms"), + success=tc.get("success", True), + error=tc.get("error"), + ) + for tc in s.get("tool_calls", []) + ] + steps.append(AgentStep( + step_type=str(s.get("step_number", "")), + planner_output=llm.get("response", ""), + tool_calls=tool_calls, + context="", + latency_ms=s.get("latency_ms"), + )) + + telemetry = incident.get("telemetry", {}) + + # Derive model from the incident trace if available + traced_model = incident.get("model") + if not traced_model: + # Fallback: try to get from first step's llm_response + for s in incident.get("agent_steps", []): + llm = s.get("llm_response", {}) + if isinstance(llm.get("model"), str): + traced_model = llm["model"] + break + model_name = traced_model or "gemini-3.1-flash-lite" + + return IncidentSnapshot( + incident_id=incident.get("id", incident.get("execution_id", "unknown")), + execution_trace=ExecutionTrace( + agent_steps=steps, + model=model_name, + total_latency_ms=telemetry.get("total_latency_ms"), + tokens_used=telemetry.get("total_prompt_tokens", 0) or 0, + ), + evaluation=EvaluationResult( + failure_modes=failure_modes, + confidence=eval_data.get("confidence", 0.5), + reasoning=eval_data.get("suspected_root_cause", eval_data.get("summary", "")), + severity=_extract_severity(eval_data), + ), + telemetry=telemetry, + metadata={ + "classification": eval_data.get("classification", ""), + "summary": eval_data.get("summary", ""), + "suspected_components": eval_data.get("suspected_components", []), + "recommendations": eval_data.get("recommendations", []), + }, + ) + + +def validate_report(report, expected: dict) -> list[str]: + errors = [] + classification = expected.get("classification", "") + + if classification == "INSUFFICIENT_EVIDENCE": + if expected.get("must_not_suggest_fix"): + fix = (report.suggested_fix or "").strip() + if fix and fix not in ("No fix required.", "INSUFFICIENT EVIDENCE"): + errors.append(f"Expected no suggested fix, got: {fix[:60]}") + if expected.get("must_not_identify_root_cause"): + rc = (report.root_cause or "").strip() + allowed_boilerplate = {"INSUFFICIENT EVIDENCE", "Unknown.", "missing trace data", "cannot investigate", "insufficient evidence"} + if rc and rc not in allowed_boilerplate: + errors.append(f"Expected no specific root cause, got: {rc[:60]}") + max_conf = expected.get("confidence_max", 1.0) + if report.confidence > max_conf: + errors.append(f"Confidence {report.confidence} > max {max_conf}") + summary = (report.summary or "").lower() + for kw in expected.get("summary_must_contain", []): + if kw not in summary: + errors.append(f"Summary should contain '{kw}'") + + if classification == "REAL_INCIDENT": + rc = (report.root_cause or "").lower() + for kw in expected.get("root_cause_must_contain", []): + if kw not in rc: + errors.append(f"Root cause should contain '{kw}'") + comps = " ".join(report.suspected_components or []) + for kw in expected.get("suspected_components_must_contain", []): + if kw.lower() not in comps.lower(): + errors.append(f"Components should contain '{kw}'") + inv = " ".join(report.suggested_investigation or []) + for kw in expected.get("suggested_investigation_must_contain", []): + if kw.lower() not in inv.lower(): + errors.append(f"Investigation should contain '{kw}'") + tests = " ".join(report.suggested_tests or []) + for kw in expected.get("suggested_tests_must_contain", []): + if kw.lower() not in tests.lower(): + errors.append(f"Tests should contain '{kw}'") + # New assertions + ev = " ".join(report.evidence or []) + for kw in expected.get("evidence_must_contain", []): + if kw.lower() not in ev.lower(): + errors.append(f"Evidence should contain '{kw}'") + rel_files = " ".join(report.relevant_files or []) + for kw in expected.get("relevant_files_must_contain", []): + if kw.lower() not in rel_files.lower(): + errors.append(f"Relevant files should contain '{kw}'") + if report.confidence < expected.get("confidence_min", 0.0): + errors.append(f"Confidence {report.confidence} < min {expected['confidence_min']}") + if not (report.summary or "").strip(): + errors.append("REAL_INCIDENT must have a summary") + if not (report.root_cause or "").strip(): + errors.append("REAL_INCIDENT must have a root cause") + if not report.evidence: + errors.append("REAL_INCIDENT must have evidence") + + return errors + + +def run_scenario(scenario: str) -> dict: + incident_path = os.path.join(_PROJECT_ROOT, "evaluation_dataset", "incidents", scenario, "incident.json") + scenario_dir = os.path.join(_HERE, "issue-agent", scenario) + eval_path = os.path.join(scenario_dir, "evaluation.json") + expected_path = os.path.join(scenario_dir, "expected.json") + + incident = load_json(incident_path) + has_api = bool(os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")) + + if has_api and os.path.exists(eval_path): + evaluator_output = load_json(eval_path) + elif has_api: + evaluator_output = run_evaluator(incident) + os.makedirs(scenario_dir, exist_ok=True) + with open(eval_path, "w") as f: + json.dump(evaluator_output, f, indent=2) + else: + evaluator_output = {"evaluation": {}, "metadata": {}} + + snapshot = build_snapshot(incident, evaluator_output) + expected = load_json(expected_path) + + _captured_logs.clear() + + repo = LocalRepo(os.path.join(_PROJECT_ROOT, "evaluation_dataset", "incidents", scenario)) + report = run_issue_agent(snapshot, repo) + + logs = list(_captured_logs) + + if report is None: + return {"scenario": scenario, "passed": False, "errors": ["Issue agent returned None"], "logs": logs} + + errors = validate_report(report, expected) + return {"scenario": scenario, "passed": len(errors) == 0, "errors": errors, "logs": logs} + + +SCENARIOS = [ + "context-overflow", "handoff-failure", "looping", "silent-hallucination", "tool-anomaly", + "crash-loop", "critical-escalation", "example", "false-positive", "insufficient-evidence", + "mixed-labels", "rate-limit", "recurring", "transient-error", +] \ No newline at end of file diff --git a/packages/issue-agent/tests/test_mapper.py b/packages/issue-agent/tests/test_mapper.py new file mode 100644 index 0000000..f1e6f34 --- /dev/null +++ b/packages/issue-agent/tests/test_mapper.py @@ -0,0 +1,128 @@ +import unittest +import os +import sys + +_HERE = os.path.dirname(__file__) +if _HERE not in sys.path: + sys.path.insert(0, _HERE) + +from test_helpers import build_snapshot, validate_report, load_json + + +_HERE = os.path.dirname(__file__) +_PROJECT_ROOT = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) + + +class TestMapper(unittest.TestCase): + def test_context_overflow_maps_correctly(self): + path = os.path.join(_PROJECT_ROOT, "evaluation_dataset", "incidents", "context-overflow", "incident.json") + incident = load_json(path) + evaluator = { + "evaluation": { + "failure_modes": ["HALLUCINATION", "SILENT_CONTEXT_OVERFLOW"], + "confidence": 1.0, + "suspected_root_cause": "context overflow", + "classification": "REAL_INCIDENT", + } + } + snapshot = build_snapshot(incident, evaluator) + self.assertEqual(snapshot.incident_id, "ctx-001") + self.assertEqual(len(snapshot.execution_trace.agent_steps), 3) + self.assertEqual(len(snapshot.execution_trace.agent_steps[0].tool_calls), 1) + self.assertEqual(snapshot.execution_trace.agent_steps[0].tool_calls[0].name, "retrieve_docs") + self.assertIn("SILENT_CONTEXT_OVERFLOW", snapshot.evaluation.failure_modes) + + def test_insufficient_evidence_maps_correctly(self): + path = os.path.join(_PROJECT_ROOT, "evaluation_dataset", "incidents", "insufficient-evidence", "incident.json") + incident = load_json(path) + evaluator = { + "evaluation": { + "failure_modes": [], + "confidence": 0.5, + "suspected_root_cause": "missing trace data", + "classification": "INSUFFICIENT_EVIDENCE", + } + } + snapshot = build_snapshot(incident, evaluator) + self.assertEqual(snapshot.incident_id, "ie-001") + + def test_validate_real_incident_pass(self): + expected = { + "classification": "REAL_INCIDENT", + "root_cause_must_contain": ["context"], + "confidence_min": 0.5, + } + report = type("R", (), { + "root_cause": "Context overflow detected", "summary": "test", + "suspected_components": ["context_manager"], + "suggested_investigation": ["prune context"], + "suggested_tests": ["regression"], + "evidence": ["tool call failed"], + "confidence": 0.9, + "suggested_fix": "", + "relevant_files": [], + })() + errors = validate_report(report, expected) + self.assertEqual(errors, []) + + def test_validate_real_incident_fail(self): + expected = { + "classification": "REAL_INCIDENT", + "root_cause_must_contain": ["context"], + "confidence_min": 0.9, + } + report = type("R", (), { + "root_cause": "something else", "summary": "test", + "suspected_components": [], + "suggested_investigation": [], + "suggested_tests": [], + "evidence": ["tool call failed"], + "confidence": 0.5, + "suggested_fix": "", + "relevant_files": [], + })() + errors = validate_report(report, expected) + self.assertGreater(len(errors), 0) + + def test_validate_insufficient_evidence_pass(self): + expected = { + "classification": "INSUFFICIENT_EVIDENCE", + "must_not_suggest_fix": True, + "must_not_identify_root_cause": True, + "confidence_max": 0.6, + "summary_must_contain": ["cannot", "trace"], + } + report = type("R", (), { + "root_cause": "missing trace data", "summary": "cannot investigate without trace data", + "suspected_components": [], + "suggested_investigation": [], + "suggested_tests": [], + "evidence": [], + "confidence": 0.3, + "suggested_fix": "", + "relevant_files": [], + })() + errors = validate_report(report, expected) + self.assertEqual(errors, []) + + def test_validate_insufficient_evidence_fails_on_fix(self): + expected = { + "classification": "INSUFFICIENT_EVIDENCE", + "must_not_suggest_fix": True, + } + report = type("R", (), { + "root_cause": "", "summary": "", + "suspected_components": [], + "suggested_investigation": [], + "suggested_tests": [], + "evidence": [], + "confidence": 0.3, + "suggested_fix": "Implement exponential backoff", + "relevant_files": [], + })() + errors = validate_report(report, expected) + self.assertGreater(len(errors), 0) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/packages/issue-agent/tests/test_repository.py b/packages/issue-agent/tests/test_repository.py new file mode 100644 index 0000000..58036c1 --- /dev/null +++ b/packages/issue-agent/tests/test_repository.py @@ -0,0 +1,92 @@ +import unittest +import os +from unittest.mock import patch, MagicMock, PropertyMock +from issue_agent.repository import GitHubRepo + + +class TestGitHubRepo(unittest.TestCase): + def setUp(self): + self.repo = GitHubRepo(token="test_token", repo="test_owner/test_repo") + self.repo._client = MagicMock() + + def test_search_symbol_returns_matching_files(self): + mock_resp = MagicMock() + mock_resp.is_error = False + mock_resp.json.return_value = { + "tree": [ + {"path": "src/tool_executor.py", "type": "blob", "sha": "abc"}, + {"path": "src/planner.py", "type": "blob", "sha": "def"}, + {"path": "README.md", "type": "blob", "sha": "ghi"}, + ] + } + self.repo._client.get.return_value = mock_resp + # Mock read_file to return content containing "executor" + self.repo.read_file = MagicMock(return_value="def executor(): pass") + results = self.repo.search_symbol("executor") + self.assertEqual(results["matches"][0]["path"], "src/tool_executor.py") + + def test_search_symbol_finds_content_in_non_matching_filename(self): + search_resp = MagicMock() + search_resp.is_error = False + search_resp.json.return_value = { + "items": [{"path": "src/services.py", "name": "services.py"}], + "incomplete_results": False, + } + tree_resp = MagicMock() + tree_resp.is_error = False + tree_resp.json.return_value = { + "tree": [ + {"path": "src/services.py", "type": "blob", "sha": "abc"}, + {"path": "README.md", "type": "blob", "sha": "def"}, + ], + "truncated": False, + } + self.repo._client.get.side_effect = [search_resp, tree_resp] + results = self.repo.search_symbol("OrderService") + self.assertEqual(len(results["matches"]), 1) + self.assertEqual(results["matches"][0]["path"], "src/services.py") + self.assertEqual(results["matches"][0]["matched_in"], "content") + + def test_search_symbol_empty_on_error(self): + mock_resp = MagicMock() + mock_resp.is_error = True + mock_resp.status_code = 500 + self.repo._client.get.return_value = mock_resp + results = self.repo.search_symbol("anything") + self.assertEqual(results["matches"], []) + + def test_build_code_graph_empty_file_list(self): + graph = self.repo.build_code_graph([]) + self.assertEqual(len(graph.nodes), 0) + + def test_create_issue_returns_url(self): + mock_resp = MagicMock() + mock_resp.is_error = False + mock_resp.json.return_value = {"html_url": "https://github.com/test_owner/test_repo/issues/1"} + self.repo._client.post.return_value = mock_resp + url = self.repo.create_issue("test title", "test body") + self.assertEqual(url, "https://github.com/test_owner/test_repo/issues/1") + + def test_create_issue_returns_none_on_error(self): + mock_resp = MagicMock() + mock_resp.is_error = True + self.repo._client.post.return_value = mock_resp + url = self.repo.create_issue("test title", "test body") + self.assertIsNone(url) + + +class TestGitHubRepoInitialization(unittest.TestCase): + def test_defaults_from_env(self): + with unittest.mock.patch.dict(os.environ, {"GITHUB_TOKEN": "env_token", "DEMO_REPOSITORY": "env_owner/env_repo"}, clear=True): + repo = GitHubRepo() + self.assertEqual(repo.token, "env_token") + self.assertEqual(repo.repo, "env_owner/env_repo") + + def test_explicit_values_override_env(self): + repo = GitHubRepo(token="explicit", repo="explicit/repo") + self.assertEqual(repo.token, "explicit") + self.assertEqual(repo.repo, "explicit/repo") + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/packages/issue-agent/tests/test_schemas.py b/packages/issue-agent/tests/test_schemas.py new file mode 100644 index 0000000..b75bcb8 --- /dev/null +++ b/packages/issue-agent/tests/test_schemas.py @@ -0,0 +1,57 @@ +import unittest +from pydantic import ValidationError +from issue_agent.schemas import ( + IncidentSnapshot, Evidence, EngineeringReport, + CodeGraph, CodeGraphNode, CodeGraphEdge, + GitHubIssueInput, +) + + +class TestSchemas(unittest.TestCase): + def test_incident_snapshot_defaults(self): + snapshot = IncidentSnapshot( + incident_id="test_001", + execution_trace={"agent_steps": []}, + evaluation={ + "failure_modes": ["TEST"], + "confidence": 0.9, + "reasoning": "test", + }, + ) + self.assertEqual(snapshot.incident_id, "test_001") + self.assertEqual(snapshot.evaluation.confidence, 0.9) + + def test_evidence_confidence_bounds(self): + with self.assertRaises(ValidationError): + Evidence(failure_mode="X", summary="test", confidence=1.5) + + def test_engineering_report_defaults(self): + report = EngineeringReport( + summary="test", + root_cause="test", + suggested_fix="test fix", + confidence=0.85, + ) + self.assertEqual(report.suspected_components, []) + + def test_code_graph_empty(self): + graph = CodeGraph() + self.assertEqual(len(graph.nodes), 0) + self.assertEqual(len(graph.edges), 0) + + def test_code_graph_with_nodes(self): + node = CodeGraphNode(file_path="test.py", kind="file") + graph = CodeGraph(nodes=[node]) + self.assertEqual(len(graph.nodes), 1) + + def test_github_issue_input(self): + issue = GitHubIssueInput(owner="o", repo="r", title="t", body="b") + self.assertEqual(issue.title, "t") + + def test_engineering_report_confidence_bounds(self): + with self.assertRaises(ValidationError): + EngineeringReport(summary="x", root_cause="x", suggested_fix="x", confidence=-0.1) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file