Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion internal/model/discover.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,11 @@ func buildDiscoveredProvider(ep inference.EndpointInfo) DiscoveredProvider {
if m.ID == "" {
continue
}
entries = append(entries, buildCustomEndpointEntry(m.ID, cluster, ""))
// The /v1 suffix is required — LiteLLM's OpenAI provider does not
// append it (CLAUDE.md pitfall 6). Without it, discovered vLLM
// endpoints 404 on /chat/completions and poison the model group
// alongside any correct `model setup custom` entry.
entries = append(entries, buildCustomEndpointEntry(m.ID, cluster+"/v1", ""))
}

return DiscoveredProvider{
Expand Down
5 changes: 3 additions & 2 deletions internal/model/discover_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,9 @@ func TestBuildDiscoveredProvider_TranslatesHost(t *testing.T) {
if first.LiteLLMParams.Model != "openai/meta-llama/Llama-3.1-8B-Instruct" {
t.Errorf("LiteLLMParams.Model = %q, want openai/meta-llama/Llama-3.1-8B-Instruct", first.LiteLLMParams.Model)
}
if first.LiteLLMParams.APIBase != "http://host.k3d.internal:8000" {
t.Errorf("APIBase = %q, want http://host.k3d.internal:8000", first.LiteLLMParams.APIBase)
// /v1 suffix required — LiteLLM's OpenAI provider does not append it.
if first.LiteLLMParams.APIBase != "http://host.k3d.internal:8000/v1" {
t.Errorf("APIBase = %q, want http://host.k3d.internal:8000/v1", first.LiteLLMParams.APIBase)
}
// buildCustomEndpointEntry sets api_key to "none" when no key is given.
if first.LiteLLMParams.APIKey != "none" {
Expand Down
20 changes: 6 additions & 14 deletions internal/model/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -1574,6 +1574,12 @@ func buildCustomEndpointEntry(modelName, clusterEndpoint, apiKey string) ModelEn
return buildCustomEndpointEntryWithOptions(modelName, clusterEndpoint, apiKey, CustomEndpointOptions{})
}

// INVARIANT: clusterEndpoint must be the full OpenAI-compatible base
// INCLUDING the /v1 suffix — LiteLLM's `openai/` provider sends requests
// to <api_base>/chat/completions verbatim and never appends /v1
// (CLAUDE.md pitfall 6). Both callers uphold this: AddCustomEndpoint
// validates by probing <endpoint>/chat/completions, and discovery only
// registers endpoints that answered <base>/v1/models (and appends /v1).
func buildCustomEndpointEntryWithOptions(modelName, clusterEndpoint, apiKey string, options CustomEndpointOptions) ModelEntry {
entry := ModelEntry{
ModelName: modelName,
Expand Down Expand Up @@ -1696,20 +1702,6 @@ func decodeBase64(s string) (string, error) {
return string(decoded[:n]), nil
}

// WarnAndStripV1Suffix checks if an endpoint URL has a trailing /v1 suffix,
// warns the user, and returns the stripped URL. For OpenAI-compatible providers,
// LiteLLM auto-appends /v1, causing double /v1/v1 if the user includes it.
func WarnAndStripV1Suffix(endpoint string) string {
trimmed := strings.TrimRight(endpoint, "/")
if strings.HasSuffix(trimmed, "/v1") {
fmt.Printf(" Warning: stripping trailing /v1 from endpoint URL (LiteLLM adds it automatically)\n")
fmt.Printf(" %s → %s\n", trimmed, strings.TrimSuffix(trimmed, "/v1"))

return strings.TrimSuffix(trimmed, "/v1")
}

return endpoint
}

// localhostToClusterEndpoint translates localhost URLs to k3d-internal URLs
// so that services running on the host are reachable from inside the k3d cluster.
Expand Down
Loading