Add provider smokes and model preference

cmd/obol/model.go

@@ -27,6 +27,7 @@ func modelCommand(cfg *config.Config) *cli.Command {
 			modelSyncCommand(cfg),
 			modelPullCommand(),
 			modelListCommand(cfg),
+			modelPreferCommand(cfg),
 			modelRemoveCommand(cfg),
 		},
 	}
@@ -213,6 +214,12 @@ func setupCloudProvider(cfg *config.Config, u *ui.UI, provider, apiKey string, m
 	u.Print("")
 	u.Successf("Model configured. To change later, run: obol model setup (or obol model remove <name>)")
 
+	if len(models) > 0 {
+		if err := model.PreferModel(cfg, u, models[0]); err != nil {
+			u.Warnf("Could not prefer configured model %q: %v", models[0], err)
+		}
+	}
+
 	return syncAgentModels(cfg, u)
 }
 
@@ -235,6 +242,28 @@ func modelSyncCommand(cfg *config.Config) *cli.Command {
 	}
 }
 
+func modelPreferCommand(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "prefer",
+		Usage:     "Move a configured model to the front of the LiteLLM preference order",
+		ArgsUsage: "<model-name>",
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			u := getUI(cmd)
+
+			modelName := cmd.Args().First()
+			if modelName == "" {
+				return errors.New("model name is required\n\nUsage: obol model prefer <model-name>\n\nList configured models with: obol model list")
+			}
+
+			if err := model.PreferModel(cfg, u, modelName); err != nil {
+				return err
+			}
+
+			return syncAgentModels(cfg, u)
+		},
+	}
+}
+
 func modelSetupCustomCommand(cfg *config.Config) *cli.Command {
 	return &cli.Command{
 		Name:  "custom",

cmd/obol/model_test.go

@@ -21,6 +21,7 @@ func TestModelCommand_Structure(t *testing.T) {
 		"sync":   false,
 		"pull":   false,
 		"list":   false,
+		"prefer": false,
 		"remove": false,
 	}
 

flows/flow-12-agent-provider-smokes.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+# Flow 12: obol-agent provider inference smokes.
+# Tests local Ollama, optional Anthropic/OpenAI provider setup, `obol model prefer`,
+# and stack-managed obol-agent inference after each preference change.
+source "$(dirname "$0")/lib.sh"
+
+provider_models() {
+    local provider="$1"
+    local status_json
+    status_json=$("$OBOL" -o json model status 2>/dev/null || true)
+    python3 -c '
+import json
+import sys
+
+provider = sys.argv[1]
+try:
+    data = json.load(sys.stdin)
+except Exception:
+    sys.exit(0)
+
+for item in data.get("providers", []):
+    if item.get("name") == provider:
+        for model in item.get("models", []):
+            if "*" not in model:
+                print(model)
+        break
+' "$provider" <<< "$status_json"
+}
+
+agent_namespace() {
+    local ns
+    ns=$("$OBOL" openclaw list 2>/dev/null | grep -oE 'openclaw-[a-z0-9-]+' | head -1 || true)
+    if [ -n "$ns" ]; then
+        echo "$ns"
+    else
+        echo "openclaw-obol-agent"
+    fi
+}
+
+agent_token() {
+    "$OBOL" openclaw token obol-agent 2>/dev/null || "$OBOL" openclaw token default 2>/dev/null || true
+}
+
+agent_primary_model() {
+    local config_json
+    config_json=$("$OBOL" kubectl get cm openclaw-config -n openclaw-obol-agent \
+        -o jsonpath='{.data.openclaw\.json}' 2>/dev/null || true)
+    python3 -c '
+import json
+import sys
+
+try:
+    data = json.load(sys.stdin)
+    print(data.get("agents", {}).get("defaults", {}).get("model", {}).get("primary", ""))
+except Exception:
+    pass
+' <<< "$config_json"
+}
+
+verify_agent_primary() {
+    local model_name="$1"
+    local primary
+    step "obol-agent primary model is $model_name"
+    primary=$(agent_primary_model)
+    if [ "$primary" = "openai/$model_name" ]; then
+        pass "obol-agent primary model: $primary"
+    else
+        fail "obol-agent primary model mismatch: ${primary:-empty} (expected openai/$model_name)"
+    fi
+}
+
+smoke_agent_inference() {
+    local label="$1"
+    local model_name="$2"
+    local ns token port pf_pid out
+
+    verify_agent_primary "$model_name"
+
+    step "$label obol-agent chat completions"
+    ns=$(agent_namespace)
+    token=$(agent_token)
+    if [ -z "$token" ]; then
+        fail "$label missing obol-agent token"
+        return 0
+    fi
+
+    port=$(pick_free_port)
+    "$OBOL" kubectl port-forward -n "$ns" svc/openclaw "$port:18789" >/dev/null 2>&1 &
+    pf_pid=$!
+
+    for _ in $(seq 1 20); do
+        if curl -sf --max-time 2 "http://localhost:$port/health" >/dev/null 2>&1; then
+            break
+        fi
+        sleep 1
+    done
+
+    out=$(curl -sf --max-time 180 -X POST "http://localhost:$port/v1/chat/completions" \
+        -H "Content-Type: application/json" \
+        -H "Authorization: Bearer $token" \
+        -d '{"model":"openclaw","messages":[{"role":"user","content":"What is 2+2? Reply with the number only."}],"max_tokens":20,"stream":false}' 2>&1) || true
+
+    cleanup_pid "$pf_pid"
+
+    if echo "$out" | grep -q "choices"; then
+        pass "$label obol-agent inference returned choices"
+    else
+        fail "$label obol-agent inference failed — ${out:0:300}"
+    fi
+}
+
+prefer_and_smoke() {
+    local label="$1"
+    local model_name="$2"
+    run_step "$label prefer $model_name" "$OBOL" model prefer "$model_name"
+    smoke_agent_inference "$label" "$model_name"
+}
+
+skip_or_fail_cloud() {
+    local provider="$1"
+    local env_var="$2"
+    if [ "${FLOW_REQUIRE_CLOUD_PROVIDERS:-false}" = "true" ]; then
+        fail "$provider smoke requires $env_var"
+    else
+        pass "$provider smoke skipped; set $env_var to enable"
+    fi
+}
+
+run_step "obol agent init" "$OBOL" agent init
+
+step "Local Ollama model configured in LiteLLM"
+local_models=$(provider_models ollama)
+local_model=$(printf '%s\n' "$local_models" | sed '/^$/d' | sed -n '2p')
+if [ -z "$local_model" ]; then
+    local_model=$(printf '%s\n' "$local_models" | sed '/^$/d' | sed -n '1p')
+fi
+
+if [ -n "$local_model" ]; then
+    pass "Local model selected for preference smoke: $local_model"
+    prefer_and_smoke "local" "$local_model"
+else
+    fail "No local Ollama model configured in LiteLLM"
+fi
+
+step "Anthropic provider smoke availability"
+if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
+    anthropic_model="${FLOW_ANTHROPIC_MODEL:-claude-sonnet-4-6}"
+    pass "ANTHROPIC_API_KEY present; testing $anthropic_model"
+    run_step "Configure Anthropic model" "$OBOL" model setup --provider anthropic --api-key "$ANTHROPIC_API_KEY" --model "$anthropic_model"
+    prefer_and_smoke "anthropic" "$anthropic_model"
+else
+    skip_or_fail_cloud "Anthropic" "ANTHROPIC_API_KEY"
+fi
+
+step "OpenAI provider smoke availability"
+if [ -n "${OPENAI_API_KEY:-}" ]; then
+    openai_model="${FLOW_OPENAI_MODEL:-gpt-4.1}"
+    pass "OPENAI_API_KEY present; testing $openai_model"
+    run_step "Configure OpenAI model" "$OBOL" model setup --provider openai --api-key "$OPENAI_API_KEY" --model "$openai_model"
+    prefer_and_smoke "openai" "$openai_model"
+else
+    skip_or_fail_cloud "OpenAI" "OPENAI_API_KEY"
+fi
+
+emit_metrics

flows/release-smoke.sh

@@ -58,6 +58,7 @@ append_report_footer() {
 
 - The runner uses the real \`obol\` CLI and the flow scripts as black-box release checks.
 - Any \`FAIL:\` line is release-gating, even when a child script exits zero.
+- \`flow-12-agent-provider-smokes.sh\` always checks local obol-agent inference and only runs Anthropic/OpenAI smokes when their API key env vars are present.
 - \`flow-11-dual-stack.sh\` writes on-chain receipt artifacts under \`$ARTIFACT_DIR/flow-11-receipts\`.
 - Set \`RELEASE_SMOKE_INCLUDE_OBOL=true\` to run \`flow-12-obol-payment.sh\`, which requires a current x402-rs facilitator binary.
 EOF
@@ -142,6 +143,7 @@ main() {
         "$SCRIPT_DIR/flow-10-anvil-facilitator.sh"
         "$SCRIPT_DIR/flow-08-buy.sh"
         "$SCRIPT_DIR/flow-09-lifecycle.sh"
+        "$SCRIPT_DIR/flow-12-agent-provider-smokes.sh"
     )
 
     for flow in "${flows[@]}"; do

internal/hermes/hermes.go

@@ -997,7 +997,7 @@ func syncObolSkills(cfg *config.Config, id string) error {
 func configuredModels(cfg *config.Config, u *ui.UI) ([]string, string, error) {
 	models, err := model.GetConfiguredModels(cfg)
 	if err == nil && len(models) > 0 {
-		primary, _ := rankModels(models)
+		primary, _ := rankModels(cfg, models)
 		return models, primary, nil
 	}
 
@@ -1030,7 +1030,7 @@ func configuredModels(cfg *config.Config, u *ui.UI) ([]string, string, error) {
 		}
 	}
 
-	primary, _ := rankModels(names)
+	primary, _ := rankModels(cfg, names)
 	return names, primary, nil
 }
 
@@ -1190,17 +1190,18 @@ func litellmMasterKey(cfg *config.Config) string {
 	return "sk-obol-" + strings.TrimSpace(string(data))
 }
 
-// rankModels delegates to model.Rank, which knows how to prefer larger local
-// models and frontier cloud models. Kept as a thin wrapper so call sites
-// don't need to import internal/model directly.
+// rankModels delegates to model.RankWithPreference, which honors any
+// explicit `obol model prefer` choice and otherwise falls through to
+// capability-aware ranking. Kept as a thin wrapper so call sites don't
+// need to import internal/model directly.
 //
 // IMPORTANT: do NOT pre-strip provider prefixes here. model.Rank strips
 // internally for ranking heuristics but returns the ORIGINAL strings so the
 // agent can round-trip them back to LiteLLM. Stripping at this layer would
 // break that round-trip — that's exactly the double-strip bug that
 // ca820c9 worked around for custom endpoints.
-func rankModels(models []string) (primary string, fallbacks []string) {
-	return model.Rank(models)
+func rankModels(cfg *config.Config, models []string) (primary string, fallbacks []string) {
+	return model.RankWithPreference(models, model.ReadPreference(cfg))
 }
 
 func k3dNodeExec(cfg *config.Config, hostPath, shellCmd string) error {

internal/hermes/rankmodels_test.go

@@ -13,7 +13,7 @@ import "testing"
 // be able to round-trip the returned primary back to LiteLLM without
 // modification.
 func TestRankModels_HermesWrapper_PrefersLargerLocalModel(t *testing.T) {
-	primary, fallbacks := rankModels([]string{
+	primary, fallbacks := rankModels(nil, []string{
 		"llama3.2:1b",
 		"qwen3.5:9b",
 		"llama3.2:3b",
@@ -31,7 +31,7 @@ func TestRankModels_HermesWrapper_PrefersLargerLocalModel(t *testing.T) {
 // `claude-opus-4-7`, not `anthropic/claude-opus-4-7`), and the wrapper must
 // preserve that.
 func TestRankModels_HermesWrapper_PrefersClaudeOverLocal(t *testing.T) {
-	primary, _ := rankModels([]string{
+	primary, _ := rankModels(nil, []string{
 		"qwen3.5:9b",
 		"claude-opus-4-7",
 		"llama3.2:1b",
@@ -49,7 +49,7 @@ func TestRankModels_HermesWrapper_PrefersClaudeOverLocal(t *testing.T) {
 // was the double-strip bug fixed in ca820c9 — this test guards against
 // reintroducing it.
 func TestRankModels_HermesWrapper_PreservesProviderPrefixIfPresent(t *testing.T) {
-	primary, _ := rankModels([]string{
+	primary, _ := rankModels(nil, []string{
 		"anthropic/claude-opus-4-7",
 		"openai/gpt-4o",
 		"qwen3.5:9b",
@@ -65,7 +65,7 @@ func TestRankModels_HermesWrapper_PreservesProviderPrefixIfPresent(t *testing.T)
 // LiteLLM with a key that no longer matched the registered route.
 func TestRankModels_HermesWrapper_CustomNamespacedEntryRoundTrips(t *testing.T) {
 	in := []string{"custom/spark1-vllm/qwen36-fast"}
-	primary, _ := rankModels(in)
+	primary, _ := rankModels(nil, in)
 	if primary != in[0] {
 		t.Fatalf("primary: got %q, want %q (must round-trip unchanged)", primary, in[0])
 	}