fix: skip mode

Author: Sean Roberts

src/config/validator.ts

@@ -89,6 +89,10 @@ export function validateScenario(data: unknown, filePath: string): asserts data
     throw new Error(`Invalid scenario at ${filePath}: "rubric" must be a string or array`);
   }
 
+  if (obj.skip !== undefined && typeof obj.skip !== "boolean") {
+    throw new Error(`Invalid scenario at ${filePath}: "skip" must be a boolean`);
+  }
+
   if (obj.agents !== undefined) {
     if (!Array.isArray(obj.agents) || obj.agents.length === 0) {
       throw new Error(`Invalid scenario at ${filePath}: "agents" must be a non-empty array of strings`);

src/runner/runner.ts

@@ -82,13 +82,24 @@ export async function run(options: RunOptions = {}): Promise<RunOutput> {
   // --- Discovery phase ---
   const jobs: Job[] = [];
   const agents = normalizeAgents(config.agents);
+  const skippedKeys = new Set<string>();
 
   for (const { name: agentName, config: agentConfig } of agents) {
     if (options.agentFilter?.length && !options.agentFilter.includes(agentName)) {
       continue;
     }
 
-    const scenarios = await discoverScenarios(configDir, config.scenarios, agentConfig.scenarios);
+    const allScenarios = await discoverScenarios(configDir, config.scenarios, agentConfig.scenarios);
+
+    // Partition into active and skipped
+    const scenarios: Scenario[] = [];
+    for (const s of allScenarios) {
+      if (s.skip) {
+        skippedKeys.add(s.key);
+      } else {
+        scenarios.push(s);
+      }
+    }
 
     const filteredScenarios = options.scenarioFilter?.length
       ? scenarios.filter((s) => options.scenarioFilter!.some((f) => f === s.key))
@@ -103,6 +114,10 @@ export async function run(options: RunOptions = {}): Promise<RunOutput> {
     }
   }
 
+  if (skippedKeys.size > 0) {
+    logger.info(`Skipped ${skippedKeys.size} scenario${skippedKeys.size > 1 ? "s" : ""}: ${[...skippedKeys].join(", ")}`);
+  }
+
   if (jobs.length === 0) {
     logger.info("No jobs discovered.");
     return buildOutput(runStart, []);

src/types/scenario.ts

@@ -2,6 +2,8 @@ export interface Scenario {
   /** Stable identifier derived from file path relative to scenarios root, sans .json */
   key: string;
   name: string;
+  /** When true, the scenario is excluded from runs. */
+  skip?: boolean;
   setup?: LifecycleAction[];
   prompt: string;
   rubric: string | RubricCriterion[];