Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core
Submodule core updated 1 files
+10 −2 llm_policy.lua
9 changes: 7 additions & 2 deletions providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -254,14 +254,19 @@ def native_adapter_handlers(timeout_s: float) -> "dict[str, Any]":


def _price_multiplier_knob(provider_id: str) -> dict:
# Default 1.0 (no nudge) for every provider: a routing preference is an
# operator decision, set + persisted from the Config tab, not hardcoded here
# (e.g. bedrock < 1.0 to prefer prepaid credits). A REAL per-call surcharge is
# NOT a multiplier — it belongs in the provider's reported/list price so it
# ranks AND bills; this lever is ranking-only (billing divides it back out).
return {
"provider": provider_id, "type": "float", "default": 1.0,
"min": 0.1, "max": 100.0, "label": "Ranking price multiplier",
"help": "A FICTITIOUS routing lever: scales this provider's price for "
"RANKING only (< 1 = prefer it, > 1 = avoid it). It does NOT change "
"billing — cost_usd always settles at the real reported cost or the "
"raw list price. 1.0 = no nudge. Marketplace/offer prices are the "
"live market and are not scaled."}
"raw list price. 1.0 = no nudge. Marketplace/offer prices keep their "
"raw quote and expose separate effective prices for ranking."}


def provider_knob_schema() -> "dict[str, dict]":
Expand Down
34 changes: 31 additions & 3 deletions serve.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,18 +182,45 @@ def make_discover_hook(registry):
Called from Lua inside rank — must be fast and never raise."""
import time

import settings

by_discovery_id = {}
for source in registry:
offers_sync = getattr(source, "offers_sync", None)
if offers_sync is None:
continue
for pid in source.provider_ids:
by_discovery_id[pid] = offers_sync
by_discovery_id[pid] = (offers_sync, source.name)

def _multiplier(provider_id: str, source_name: str) -> float:
for key in (f"{provider_id}.price_multiplier",
f"{source_name}.price_multiplier"):
if key in settings.SCHEMA:
return settings.get(key)
return 1.0

def _effective_offer(offer: dict, provider_id: str, source_name: str) -> dict:
mult = _multiplier(provider_id, source_name)
if mult == 1.0:
return offer
out = dict(offer)
out["ranking_price_multiplier"] = mult
for raw_key, effective_key in (
("price_in_usd_per_mtok", "effective_price_in_usd_per_mtok"),
("price_out_usd_per_mtok", "effective_price_out_usd_per_mtok"),
):
raw = offer.get(raw_key)
try:
out[effective_key] = float(raw) * mult
except (TypeError, ValueError):
pass
return out

def hook(discovery_id):
fn = by_discovery_id.get(discovery_id)
if fn is None:
entry = by_discovery_id.get(discovery_id)
if entry is None:
return {"ok": False, "error": "unknown discovery_id"}
fn, source_name = entry
try:
offers = fn(discovery_id)
except Exception as exc: # noqa: BLE001
Expand All @@ -203,6 +230,7 @@ def hook(discovery_id):
# TTL — a router that starts before the first market dump should
# pick offers up on the next rank, not minutes later.
return {"ok": False, "error": "no offers"}
offers = [_effective_offer(o, discovery_id, source_name) for o in offers]
return {"ok": True, "fetched_at_ms": int(time.time() * 1000),
"offers": offers}

Expand Down
32 changes: 21 additions & 11 deletions shim.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,11 +261,19 @@ def reload_codex_accounts():
@app.post("/x/config/reload")
def reload_config():
"""Re-read operator config overrides (dashboard Config tab) so tunable
knobs (antseed top-N, codex scarcity ramp, runway thresholds) apply
without a router restart. Sources read settings.get live, so a reload is
enough. Internal — /x/* is hidden from consumers."""
knobs (antseed top-N, codex scarcity ramp, runway thresholds, price
multipliers) apply without a router restart. Sources read settings.get
live; marketplace discovery is invalidated so effective offer prices
refresh immediately instead of waiting for the discovery TTL. Internal
— /x/* is hidden from consumers."""
import settings as _settings
return {"ok": True, "overrides": _settings.reload()}
overrides = _settings.reload()
for provider in (host.catalog().get("providers") or {}).values():
if isinstance(provider, dict) and provider.get("discovery") == "marketplace":
did = provider.get("discovery_id")
if did:
host.invalidate_discovery(did)
return {"ok": True, "overrides": overrides}

@app.get("/x/session/{sid}")
def session_meter(sid: str, request: Request):
Expand Down Expand Up @@ -1441,14 +1449,16 @@ def _executed_cost_usd(result: dict, subscription_providers=frozenset()) -> floa
# (3) compute from the ranker price, discounting cache-read input tokens
chosen = result.get("chosen") or {}
pin, pout = chosen.get("price_in"), chosen.get("price_out")
# The ranking price carries the provider's effective-price multiplier — a
# FICTITIOUS routing lever (providers.price_multiplier, applied in
# sources.push_prices). Billing must use the RAW list price, so divide it back
# out: cost_usd never moves with the multiplier. Reported-cost providers settle
# in step 2 and never reach here, so this keeps billing uniform across them.
# The ranking price can carry an effective-price multiplier — a FICTITIOUS
# routing lever (static prices via sources.push_prices; marketplace offers via
# discovery). Billing must use the RAW list/quote price, so divide it back out:
# cost_usd never moves with the multiplier. Reported-cost providers settle in
# step 2 and never reach here, so this keeps billing uniform across them.
import settings
mkey = f"{chosen.get('provider_id')}.price_multiplier"
mult = settings.get(mkey) if mkey in settings.SCHEMA else 1.0
mult = chosen.get("price_multiplier")
if not isinstance(mult, (int, float)) or isinstance(mult, bool):
mkey = f"{chosen.get('provider_id')}.price_multiplier"
mult = settings.get(mkey) if mkey in settings.SCHEMA else 1.0
if mult and mult > 0:
pin = pin / mult if pin is not None else pin
pout = pout / mult if pout is not None else pout
Expand Down
13 changes: 8 additions & 5 deletions sources/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,14 @@ def push_prices(host: Any, catalog: dict, prices: list[Price]) -> int:
and price-ceiling filters read). Unmapped or un-cataloged prices are
skipped — sources never widen the catalog.

Each price is scaled by the provider's `<provider>.price_multiplier` knob
(default 1.0) on the way in, so RANKING sees a nudged price (a fictitious
routing lever). Billing is unaffected: the raw list price stays untouched at
the source/table, and shim._executed_cost_usd divides the same multiplier back
out before computing cost_usd — so the lever never distorts spend."""
Static provider prices are scaled by the provider's
`<provider>.price_multiplier` knob (default 1.0) on the way in, so RANKING
sees a nudged price (a fictitious routing lever). Marketplace offers keep
raw quotes in their source cache and receive effective ranking prices in the
discover hook. Billing is unaffected: the raw list/quote price stays
untouched at the source/table, and shim._executed_cost_usd divides the same
multiplier back out before computing cost_usd — so the lever never distorts
spend."""
import settings # lazy: settings -> providers, never imports sources back
pairs = _served_pairs(catalog)
now = int(time.time())
Expand Down
40 changes: 40 additions & 0 deletions tests/test_live_wiring.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,46 @@ def test_marketplace_offers_rank_with_offer_prices():
assert ("antseed", "claude-opus-4-8") in pairs


def test_marketplace_offers_rank_with_effective_prices_when_present():
host = LLMRouterHost(
router_path=ROOT / "core" / "router.lua",
config_path=ROOT / "config.live.lua",
metrics_path=ROOT / "metrics.live.lua",
env=LIVE_TEST_ENV.copy(),
now_ms=lambda: 1,
)
host.set_discover_hook(lambda did: {
"ok": True, "fetched_at_ms": 1,
"offers": [{
"model_family": "acme/discounted",
"wire_model_id": "acme/discounted",
"seller_endpoint": "https://openrouter.ai/api/v1",
"price_in_usd_per_mtok": 10.0,
"price_out_usd_per_mtok": 20.0,
"effective_price_in_usd_per_mtok": 8.0,
"effective_price_out_usd_per_mtok": 16.0,
"ranking_price_multiplier": 0.8,
"capabilities": {"context": 200000},
"traits": {"bench_intelligence": 0.5},
}],
} if did == "openrouter_market" else {"ok": False, "error": "x"})
host.init()
term = ["policy",
["and", ["meets_req"], ["not", ["is", "disabled"]],
["family_eq", "acme/discounted"]],
["neg", ["normalize", ["field", "price_in"]]],
["argmax"], ["id"], ["always", {"action": "next_candidate"}]]

ranked, _ = host.rank({"policy_ir": term, "requirements": {"context": 8000}})

candidate = ranked[0]["candidate"]
assert candidate["price_in"] == 8.0
assert candidate["price_out"] == 16.0
assert candidate["price_multiplier"] == 0.8
assert candidate["offer"]["price_in_usd_per_mtok"] == 10.0
assert candidate["offer"]["effective_price_in_usd_per_mtok"] == 8.0


def test_discovered_family_ranks_on_inline_offer_traits():
"""A discovered family (raw model id, absent from model_meta.lua) ranks on
the live benchmark it carries inline on the offer (c.offer.traits) — the
Expand Down
5 changes: 4 additions & 1 deletion tests/test_providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,10 @@ def test_priced_providers_get_an_effective_multiplier_knob():
for p in providers.PROVIDERS:
key = f"{p.id}.price_multiplier"
if p.source is not None: # only providers that push a price
assert key in sch and sch[key]["default"] == 1.0 and sch[key]["type"] == "float"
# default is a neutral 1.0 (no nudge); a routing preference is set
# from the UI and persisted, never hardcoded here.
assert key in sch and sch[key]["default"] == 1.0 \
and sch[key]["type"] == "float"
else:
assert key not in sch # no dead knob where it can't apply

Expand Down
14 changes: 14 additions & 0 deletions tests/test_shim.py
Original file line number Diff line number Diff line change
Expand Up @@ -561,6 +561,20 @@ def test_executed_cost_usd_ignores_the_ranking_multiplier(monkeypatch):
"response": {"tokens_in": 1_000_000, "tokens_out": 100_000}}
assert _executed_cost_usd(result) == 3.0 # == raw list 2.0/10.0, not 1.5

# Marketplace providers may not have their own settings key
# (e.g. bedrock_market inherits bedrock.price_multiplier during discovery),
# so the chosen candidate carries the exact multiplier ranking used.
market = {
"chosen": {
"provider_id": "bedrock_market",
"price_in": 0.8,
"price_out": 8.0,
"price_multiplier": 0.8,
},
"response": {"tokens_in": 1_000_000, "tokens_out": 100_000},
}
assert _executed_cost_usd(market) == 2.0 # == raw quote 1.0/10.0, not 1.6


def test_cost_basis_tiers():
"""How cost_usd was determined — the raw fact the cost-accuracy panel reads to
Expand Down
36 changes: 35 additions & 1 deletion tests/test_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ def test_push_prices_only_pushes_cataloged_pairs():
assert pushed == 1
(provider, family, delta), = host.pushed
assert (provider, family) == ("openrouter", "gpt-5.5")
assert delta["price_in"] == 5.0 and delta["price_out"] == 30.0 # multiplier 1.0 default
# default multiplier is 1.0 → the ranking price equals the raw list price
assert delta["price_in"] == 5.0 and delta["price_out"] == 30.0
assert isinstance(delta["price_refreshed_at"], int)


Expand Down Expand Up @@ -1169,6 +1170,39 @@ def test_discover_hook_serves_antseed_offers(tmp_path):
assert r2["ok"] is False and "no offers" in r2["error"]


def test_discover_hook_adds_effective_offer_prices_without_mutating_raw(monkeypatch):
import serve

monkeypatch.setitem(settings._overrides, "bedrock.price_multiplier", 0.8)
raw_offer = {
"model_family": "claude-sonnet-4-6",
"wire_model_id": "us.anthropic.claude-sonnet-4-6",
"seller_endpoint": "bedrock:us-east-1",
"price_in_usd_per_mtok": 3.0,
"price_out_usd_per_mtok": 15.0,
}

class FakeBedrockSource:
name = "bedrock"
provider_ids = ["bedrock_market"]

def offers_sync(self, provider_id):
assert provider_id == "bedrock_market"
return [raw_offer]

hook = serve.make_discover_hook([FakeBedrockSource()])
r = hook("bedrock_market")

assert r["ok"] is True
offer = r["offers"][0]
assert offer["price_in_usd_per_mtok"] == 3.0
assert offer["price_out_usd_per_mtok"] == 15.0
assert offer["effective_price_in_usd_per_mtok"] == pytest.approx(2.4)
assert offer["effective_price_out_usd_per_mtok"] == 12.0
assert offer["ranking_price_multiplier"] == 0.8
assert "effective_price_in_usd_per_mtok" not in raw_offer


# ---- codex passive source ---------------------------------------------------

def test_codex_source_aggregates_signals_into_quota_balance():
Expand Down