fix: reuse common workspace

Author: Sean Roberts

src/runner/runner.ts

@@ -344,6 +344,7 @@ async function executeJob(
         rubric: scenario.rubric,
         agentConfig,
         output,
+        workingDirectory: workspace,
       },
       cleanup,
     };

src/scoring/judge.ts

@@ -13,17 +13,23 @@ export interface JudgeCallOptions {
 
 /**
  * Call an LLM judge using the same adapter as the test run.
- * Creates an isolated temp workspace, runs the adapter with a synthetic scenario,
- * and returns the result text.
+ *
+ * Uses the agent's original workspace when available so the judge can
+ * independently verify the agent's actual work (files created, endpoints
+ * deployed, etc.). Falls back to a disposable temp directory only when
+ * no workspace is set (e.g. programmatic API usage without the runner).
  */
 export async function callJudge(
   runResult: RunResult,
   prompt: string,
   options: JudgeCallOptions,
 ): Promise<string> {
   const adapter = getAdapter(runResult.agentConfig.adapter);
+  const originalWorkspace = runResult.workingDirectory;
+
+  const workspace = originalWorkspace ?? fs.mkdtempSync(path.join(os.tmpdir(), `axis-${options.scenarioKey}-`));
+  const shouldCleanup = !originalWorkspace;
 
-  const workspace = fs.mkdtempSync(path.join(os.tmpdir(), `axis-${options.scenarioKey}-`));
   try {
     const output = await adapter.run({
       prompt,
@@ -38,10 +44,12 @@ export async function callJudge(
     });
     return output.result ?? "";
   } finally {
-    try {
-      fs.rmSync(workspace, { recursive: true, force: true });
-    } catch {
-      /* ignore */
+    if (shouldCleanup) {
+      try {
+        fs.rmSync(workspace, { recursive: true, force: true });
+      } catch {
+        /* ignore */
+      }
     }
   }
 }

src/scoring/prompt-templates.ts

@@ -183,7 +183,7 @@ const GOAL_STRING_RUBRIC_TEMPLATE: PromptTemplate = {
   stage: "goal_achievement",
   template: `You are an expert evaluator for an AI agent testing framework called AXIS.
 
-An AI agent was given a task. You must evaluate how well it performed by reviewing its transcript AND by independently verifying the results yourself.
+An AI agent was given a task. You must evaluate how well it performed based on the evidence in its transcript.
 
 SCENARIO: {{scenarioName}}
 
@@ -209,8 +209,7 @@ RUBRIC:
 
 INSTRUCTIONS:
 1. Review the transcript to understand what the agent did.
-2. Where possible, independently verify the results — visit URLs, check endpoints, confirm that the claimed outcomes actually exist. Do not trust the transcript alone.
-3. Score based on what you can verify, not just what the agent claims.
+2. Where possible, independently verify the results — check the filesystem for created/modified files, visit URLs, confirm that the claimed outcomes actually exist. Do not trust the transcript alone.
 
 When done, respond with ONLY valid JSON on its own line:
 {"score": <0-10>, "rationale": "<1-2 sentence explanation>"}

src/types/output.ts

@@ -20,6 +20,8 @@ export interface BaseRunResult {
   rubric: string | RubricCriterion[];
   agentConfig: AgentConfig;
   output: AgentOutput;
+  /** Path to the agent's workspace directory (available during scoring, before cleanup). */
+  workingDirectory?: string;
 }
 
 export interface RunResult extends BaseRunResult {}