feat: init support

Author: Sean Roberts

src/adapters/registry.ts

@@ -24,8 +24,8 @@ export function getAdapter(adapterName: string): AgentAdapter {
   const factory = BUILTIN_FACTORIES[adapterName];
   if (!factory) {
     throw new Error(
-      `Unknown adapter: "${adapterName}". Built-in: claude-sdk, claude-code, codex, gemini, gemini-acp, goose. ` +
-        `Register custom adapters via the "adapters" config field or registerAdapter().`,
+      `Unknown agent: "${adapterName}". Built-in: ${getBuiltinAdapterNames().join(", ")}. ` +
+        `Register custom agents via the "adapters" config field or registerAdapter().`,
     );
   }
 
@@ -34,6 +34,11 @@ export function getAdapter(adapterName: string): AgentAdapter {
   return instance;
 }
 
+/** Names of all built-in agent adapters. */
+export function getBuiltinAdapterNames(): string[] {
+  return Object.keys(BUILTIN_FACTORIES);
+}
+
 /** Register a custom adapter by name. */
 export function registerAdapter(name: string, adapter: AgentAdapter): void {
   instanceCache.set(name, adapter);

src/cli.ts

@@ -2,6 +2,7 @@
 
 import * as fs from "node:fs";
 import * as path from "node:path";
+import * as readline from "node:readline";
 import { fileURLToPath } from "node:url";
 import { Command } from "commander";
 import { run } from "./runner/runner.js";
@@ -11,6 +12,7 @@ import { initReport, finalizeReport } from "./reports/writer.js";
 import { listReports, readReport, readScenarioResults } from "./reports/reader.js";
 import { setBaseline, readBaseline, listBaselines, deleteBaseline, DEFAULT_BASELINE_NAME } from "./baselines/store.js";
 import { compareBaseline } from "./baselines/compare.js";
+import { getBuiltinAdapterNames } from "./adapters/registry.js";
 import {
   renderReportList,
   renderReportDetail,
@@ -52,6 +54,105 @@ function handleSignal(signal: NodeJS.Signals): void {
 process.on("SIGINT", () => handleSignal("SIGINT"));
 process.on("SIGTERM", () => handleSignal("SIGTERM"));
 
+// --- axis init command ---
+
+const BUILT_IN_AGENTS = ["claude-code", "codex", "gemini"];
+
+function prompt(rl: readline.Interface, question: string, defaultValue: string): Promise<string> {
+  return new Promise((resolve) => {
+    rl.question(question, (answer) => {
+      resolve(answer.trim() || defaultValue);
+    });
+  });
+}
+
+program
+  .command("init")
+  .description("Initialize a new AXIS configuration and sample scenario")
+  .option("-s, --scenarios <path>", "path to scenarios directory", "./scenarios")
+  .option("-a, --agent <names>", "agent(s) to include (comma-separated, e.g. claude-code,codex)")
+  .option("-f, --force", "overwrite existing files")
+  .action(async (opts) => {
+    let scenariosPath: string = opts.scenarios;
+    let agents: string[] = opts.agent
+      ? opts.agent
+          .split(/[\s,]+/)
+          .filter(Boolean)
+          .map((a: string) => a.toLowerCase())
+      : [];
+
+    const hasExplicitFlags = opts.agent || opts.scenarios !== "./scenarios";
+    const interactive = process.stdin.isTTY && !hasExplicitFlags;
+
+    if (interactive) {
+      const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+
+      scenariosPath = await prompt(rl, `  Scenarios directory (${scenariosPath}): `, scenariosPath);
+      const agentAnswer = await prompt(
+        rl,
+        `  Agents [${BUILT_IN_AGENTS.join(", ")}] (claude-code): `,
+        "claude-code",
+      );
+      agents = agentAnswer
+        .split(/[\s,]+/)
+        .filter(Boolean)
+        .map((a) => a.toLowerCase());
+      rl.close();
+    }
+
+    // Filter out unknown agents (warn the user about which were ignored)
+    const knownAgents = new Set(getBuiltinAdapterNames());
+    const ignored = agents.filter((a) => !knownAgents.has(a));
+    agents = agents.filter((a) => knownAgents.has(a));
+
+    if (ignored.length > 0) {
+      process.stderr.write(
+        `\n  Warning: ignoring unknown agent${ignored.length > 1 ? "s" : ""}: ${ignored.join(", ")}\n` +
+          `  Built-in agents: ${getBuiltinAdapterNames().join(", ")}\n`,
+      );
+    }
+
+    if (agents.length === 0) agents = ["claude-code"];
+
+    const configPath = path.resolve("axis.config.json");
+    const scenariosDir = path.resolve(scenariosPath);
+    const scenarioFile = path.join(scenariosDir, "hello-world.json");
+
+    // Check for existing files
+    if (!opts.force) {
+      if (fs.existsSync(configPath)) {
+        process.stderr.write("\n  axis.config.json already exists. Use --force to overwrite.\n\n");
+        process.exit(1);
+      }
+      if (fs.existsSync(scenarioFile)) {
+        process.stderr.write(`\n  ${path.relative(".", scenarioFile)} already exists. Use --force to overwrite.\n\n`);
+        process.exit(1);
+      }
+    }
+
+    const config = {
+      scenarios: scenariosPath,
+      agents,
+    };
+
+    const scenario = {
+      name: "Hello World",
+      prompt: "Create a file called hello.txt with the content 'Hello, World!'",
+      rubric: [
+        { check: "A file named hello.txt was created" },
+        { check: "The file contains the text 'Hello, World!'" },
+      ],
+    };
+
+    fs.mkdirSync(scenariosDir, { recursive: true });
+    fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n");
+    fs.writeFileSync(scenarioFile, JSON.stringify(scenario, null, 2) + "\n");
+
+    process.stdout.write(`\n  Created axis.config.json\n`);
+    process.stdout.write(`  Created ${path.relative(".", scenarioFile)}\n\n`);
+    process.stdout.write(`  Run \`axis run\` to execute your first scenario.\n\n`);
+  });
+
 // --- Shared run pipeline ---
 
 interface RunPipelineOptions {
@@ -166,7 +267,7 @@ program
     const pipelineOpts: RunPipelineOptions = {
       configPath: opts.config,
       scenario: opts.scenario,
-      agent: opts.agent,
+      agent: opts.agent ? opts.agent.toLowerCase() : undefined,
       concurrency: opts.concurrency,
       score: opts.score,
       verbose: opts.verbose,
@@ -331,7 +432,7 @@ program
 
       // View a specific scenario result
       if (scenarioKey) {
-        const agentFilter: string[] | undefined = opts.agent;
+        const agentFilter: string[] | undefined = opts.agent?.map((a: string) => a.toLowerCase());
 
         // Read all agents, then filter if --agent was specified
         let results = readScenarioResults(configDir, reportId, scenarioKey);

src/config/loader.ts

@@ -23,13 +23,26 @@ export async function loadConfig(configPath?: string): Promise<{ config: AxisCon
   }
 
   validateConfig(parsed, resolvedPath);
+  normalizeConfigAgents(parsed);
 
   return {
     config: parsed,
     configDir: path.dirname(resolvedPath),
   };
 }
 
+/** Lowercase all agent names in a validated config (mutates in place). */
+function normalizeConfigAgents(config: AxisConfig): void {
+  for (let i = 0; i < config.agents.length; i++) {
+    const entry = config.agents[i];
+    if (typeof entry === "string") {
+      config.agents[i] = entry.toLowerCase();
+    } else {
+      entry.adapter = entry.adapter.toLowerCase();
+    }
+  }
+}
+
 export async function discoverScenarios(
   configDir: string,
   scenariosPath: string,
@@ -97,6 +110,7 @@ async function loadScenarioFile(filePath: string, rootDir: string): Promise<Scen
   }
 
   validateScenario(parsed, filePath);
+  normalizeScenarioAgents(parsed);
 
   // Derive key from relative path: scenarios/cms/create-post.json → "cms/create-post"
   const relativePath = path.relative(rootDir, filePath);
@@ -116,6 +130,18 @@ async function loadScenarioFile(filePath: string, rootDir: string): Promise<Scen
   return scenario.variants.map((variant) => expandVariant(scenario, variant, baseKey));
 }
 
+/** Lowercase agent-name entries in a scenario and any variants (mutates in place). */
+function normalizeScenarioAgents(scenario: Scenario & { variants?: ScenarioVariant[] }): void {
+  if (scenario.agents) {
+    scenario.agents = scenario.agents.map((a) => a.toLowerCase());
+  }
+  if (scenario.variants) {
+    for (const v of scenario.variants) {
+      if (v.agents) v.agents = v.agents.map((a) => a.toLowerCase());
+    }
+  }
+}
+
 function expandVariant(parent: Scenario, variant: ScenarioVariant, baseKey: string): Scenario {
   const expanded: Scenario = {
     key: `${baseKey}@${variant.name}`,

src/docs-site/src/pages/cli.astro

@@ -9,6 +9,32 @@ import DocsLayout from "../layouts/DocsLayout.astro";
     directly or via <code>npx @netlify/axis</code>.
   </p>
 
+  <h2><code>axis init</code></h2>
+  <div class="cli-block">
+    <div class="cli-command">axis init [options]</div>
+    <div class="cli-desc">
+      Scaffold a new <code>axis.config.json</code> and a sample scenario in <code>./scenarios</code>. When run in
+      a TTY without flags, prompts interactively for the scenarios directory and agents.
+    </div>
+    <div class="cli-flags">
+      <div class="cli-flag">
+        <span class="cli-flag-name"><code>-s, --scenarios &lt;path&gt;</code></span>
+        <span class="cli-flag-desc">Path to scenarios directory (default: <code>./scenarios</code>).</span>
+      </div>
+      <div class="cli-flag">
+        <span class="cli-flag-name"><code>-a, --agent &lt;names&gt;</code></span>
+        <span class="cli-flag-desc">
+          Agent(s) to include — comma-separated (e.g. <code>claude-code,codex,gemini</code>). Default:
+          <code>claude-code</code>.
+        </span>
+      </div>
+      <div class="cli-flag">
+        <span class="cli-flag-name"><code>-f, --force</code></span>
+        <span class="cli-flag-desc">Overwrite existing files.</span>
+      </div>
+    </div>
+  </div>
+
   <h2><code>axis run</code></h2>
   <div class="cli-block">
     <div class="cli-command">axis run [options]</div>

src/docs-site/src/pages/quickstart.astro

@@ -5,8 +5,8 @@ import DocsLayout from "../layouts/DocsLayout.astro";
 <DocsLayout title="Quick Start" active="quickstart">
   <h1>Quick Start</h1>
   <p class="lead">
-    Get AXIS running in your project. This guide walks through creating a config,
-    writing your first scenario, running it, and understanding the results.
+    Get AXIS running in your project. Install the CLI, initialize a config, then iterate: run,
+    review, and baseline.
   </p>
 
   <h2>Prerequisites</h2>
@@ -15,52 +15,41 @@ import DocsLayout from "../layouts/DocsLayout.astro";
     <li>An API key for at least one supported agent (for example, <code>ANTHROPIC_API_KEY</code> for Claude Code).</li>
   </ul>
 
-  <h2>1. Create a Config File</h2>
+  <h2>1. Install the CLI</h2>
   <p>
-    Add an <code>axis.config.json</code> to your project root. At minimum, specify where your
-    scenarios live and which agents to run.
+    Install AXIS globally so the <code>axis</code> binary is available on your <code>PATH</code>:
   </p>
-  <pre><code>{`{
-  "scenarios": "./scenarios",
-  "agents": ["claude-code"]
-}`}</code></pre>
+  <pre><code>npm install -g @netlify/axis</code></pre>
   <p>
-    See <a href="/configuration">Configuration Reference</a> for the full set of options, including
-    scoring weights, MCP servers, and custom agents.
+    Or, to skip the install and run directly with <code>npx</code>, prefix any command with
+    <code>npx @netlify/axis</code> (for example, <code>npx @netlify/axis init</code>).
   </p>
 
-  <h2>2. Write a Scenario</h2>
+  <h2>2. Initialize Your Project</h2>
   <p>
-    Create a <code>scenarios/</code> directory and add your first scenario as a JSON file. Each
-    scenario needs three things: a <strong>name</strong>, a <strong>prompt</strong> (the task for
-    the agent), and a <strong>rubric</strong> (how to judge whether it succeeded).
-  </p>
-  <pre><code>{`{
-  "name": "Create a greeting file",
-  "prompt": "Create a file called hello.txt with the content 'Hello from AXIS'.",
-  "rubric": [
-    { "check": "File hello.txt exists in the workspace", "weight": 0.5 },
-    { "check": "File contains exactly 'Hello from AXIS'", "weight": 0.5 }
-  ]
-}`}</code></pre>
-  <p>
-    Save this as <code>scenarios/hello-world.json</code>. The filename (without <code>.json</code>)
-    becomes the scenario key used in reports and CLI commands.
+    From your project root, run:
   </p>
+  <pre><code>axis init</code></pre>
   <p>
-    A few things that make scenarios work well:
+    In an interactive terminal, this prompts you for the scenarios directory and which agents to
+    include (comma-separated, e.g. <code>claude-code,codex,gemini</code>). It then creates two files:
   </p>
   <ul>
-    <li><strong>Specific prompts</strong> -tell the agent exactly what to do. Vague prompts lead to inconsistent results.</li>
-    <li><strong>Observable rubric checks</strong> -each check should describe something a judge can verify from the transcript and workspace state.</li>
-    <li><strong>Weighted checks</strong> -distribute weight based on importance. If the file existing matters more than its content, weight it higher.</li>
+    <li><code>axis.config.json</code> -minimal config with your chosen scenarios path and agents.</li>
+    <li><code>scenarios/hello-world.json</code> -a sample scenario that asks the agent to create a file with specific content.</li>
   </ul>
   <p>
-    See <a href="/scenarios">Writing Scenarios</a> for a deeper guide on prompts, rubrics, setup/teardown, and examples.
+    To skip the prompts, pass flags directly:
+  </p>
+  <pre><code>axis init --agent claude-code,codex --scenarios ./scenarios</code></pre>
+  <p>
+    See the <a href="/configuration">Configuration Reference</a> for additional options like scoring
+    weights, MCP servers, and custom agents, and <a href="/scenarios">Writing Scenarios</a> for guidance
+    on writing effective prompts and rubrics.
   </p>
 
   <h2>3. Run It</h2>
-  <pre><code>npx @netlify/axis run</code></pre>
+  <pre><code>axis run</code></pre>
   <p>
     AXIS spawns the agent in an isolated workspace, captures the full interaction transcript, scores
     the result against your rubric, and displays a summary in your terminal.
@@ -86,13 +75,13 @@ import DocsLayout from "../layouts/DocsLayout.astro";
     Every run saves a report to <code>.axis/reports/</code>. You can revisit it at any time.
   </p>
   <pre><code>{`# View the latest report summary
-npx @netlify/axis reports latest
+axis reports latest
 
 # Open the HTML report in your browser
-npx @netlify/axis reports latest --html
+axis reports latest --html
 
 # Get JSON output for scripting
-npx @netlify/axis reports latest --json`}</code></pre>
+axis reports latest --json`}</code></pre>
   <p>
     The HTML report includes the full scoring breakdown, interaction transcript, and judge
     evaluations. See <a href="/reports">Reports & Baselines</a> for details on report contents
@@ -142,10 +131,10 @@ npx @netlify/axis reports latest --json`}</code></pre>
     it to detect regressions.
   </p>
   <pre><code>{`# Save the latest report as a baseline
-npx @netlify/axis baseline set
+axis baseline set
 
 # Compare future runs automatically
-npx @netlify/axis run --compare-baseline`}</code></pre>
+axis run --compare-baseline`}</code></pre>
   <p>
     The comparison exits with code 1 if any regressions are detected, making it suitable for CI
     gating. See <a href="/reports">Reports & Baselines</a> for baseline workflows.

src/runner/runner.ts

@@ -203,7 +203,7 @@ export async function run(options: RunOptions = {}): Promise<RunOutput> {
     const missing = required.filter((key) => !jobEnv[key]);
     if (missing.length > 0) {
       throw new Error(
-        `The "${job.agentConfig.adapter}" adapter requires environment variable${missing.length > 1 ? "s" : ""} ${missing.join(", ")} ` +
+        `The "${job.agentConfig.adapter}" agent requires environment variable${missing.length > 1 ? "s" : ""} ${missing.join(", ")} ` +
           `but ${missing.length > 1 ? "they are" : "it is"} not set. ` +
           `Add ${missing.length > 1 ? "them" : "it"} to your shell environment or to the "env" array in axis.config.json.`,
       );