feat(ubi): engine-neutral User Behavior Insights judgment generation

backend/app/agent/confirmation.py

@@ -1,7 +1,8 @@
 """Confirmation guard primitives (feat_chat_agent Story 2.5).
 
-* :data:`MUTATING_TOOL_NAMES` — the 7-tool set requiring confirmation per spec
-  FR-5 + §19 Decision log. ``create_query_set`` is intentionally NOT on this
+* :data:`MUTATING_TOOL_NAMES` — the 8-tool set requiring confirmation per spec
+  FR-5 + §19 Decision log (+ ``generate_judgments_from_ubi`` from
+  feat_ubi_judgments FR-6). ``create_query_set`` is intentionally NOT on this
   list (creating an empty container is cheap to undo).
 * :func:`is_affirmative` — whole-word, case-insensitive matcher against a small
   affirmative-token vocabulary.
@@ -15,6 +16,10 @@
     {
         "import_queries_from_csv",
         "generate_judgments_llm",
+        # feat_ubi_judgments FR-6 — UBI judgment generation is equivalent to
+        # the LLM path in operator commitment + data side-effects, so the
+        # server-side confirmation guard enforces it too (not prompt-only).
+        "generate_judgments_from_ubi",
         "create_study",
         "cancel_study",
         "create_proposal_from_study",

backend/app/agent/tools/__init__.py

@@ -43,6 +43,11 @@
     ListClustersArgs,
     list_clusters_impl,
 )
+from backend.app.agent.tools.judgments.generate_judgments_from_ubi import (
+    GENERATE_JUDGMENTS_FROM_UBI_TOOL,
+    GenerateJudgmentsFromUbiArgs,
+    generate_judgments_from_ubi_impl,
+)
 from backend.app.agent.tools.judgments.generate_judgments_llm import (
     GENERATE_JUDGMENTS_LLM_TOOL,
     GenerateJudgmentsLLMArgs,
@@ -156,6 +161,7 @@
     CREATE_QUERY_SET_TOOL,
     IMPORT_QUERIES_FROM_CSV_TOOL,
     GENERATE_JUDGMENTS_LLM_TOOL,
+    GENERATE_JUDGMENTS_FROM_UBI_TOOL,
     GET_CALIBRATION_TOOL,
     RUN_QUERY_TOOL,
     # Studies (Story 2.3 + feat_agent_propose_search_space)
@@ -183,6 +189,8 @@
     "create_query_set": create_query_set_impl,
     "import_queries_from_csv": import_queries_from_csv_impl,
     "generate_judgments_llm": generate_judgments_llm_impl,
+    # feat_ubi_judgments Story 3.4
+    "generate_judgments_from_ubi": generate_judgments_from_ubi_impl,
     "get_calibration": get_calibration_impl,
     "run_query": run_query_impl,
     # Story 2.3 + feat_agent_propose_search_space
@@ -210,6 +218,8 @@
     "create_query_set": CreateQuerySetArgs,
     "import_queries_from_csv": ImportQueriesFromCsvArgs,
     "generate_judgments_llm": GenerateJudgmentsLLMArgs,
+    # feat_ubi_judgments Story 3.4
+    "generate_judgments_from_ubi": GenerateJudgmentsFromUbiArgs,
     "get_calibration": GetCalibrationArgs,
     "run_query": RunQueryArgs,
     # Story 2.3 + feat_agent_propose_search_space

backend/app/agent/tools/judgments/generate_judgments_from_ubi.py

@@ -0,0 +1,124 @@
+"""``generate_judgments_from_ubi`` tool (feat_ubi_judgments Story 3.4 / FR-6).
+
+Mirrors ``generate_judgments_llm`` for the UBI path. Routes through the
+shared :func:`backend.app.services.agent_judgments_dispatch.start_ubi_judgment_generation`
+dispatcher — same preflight runs as the
+``POST /api/v1/judgments/generate-from-ubi`` endpoint.
+
+MUTATING tool — the orchestrator's confirmation guard requires
+affirmative user message before dispatch (per ``feat_chat_agent`` §19
+Decision log; UBI lists are equivalent to LLM lists in terms of
+operator commitment + side-effects on the operator's data).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+from uuid import UUID
+
+from openai.types.chat import ChatCompletionToolParam
+from pydantic import BaseModel, Field, model_validator
+
+from backend.app.agent.context import ToolContext
+from backend.app.api.v1.schemas import (
+    UbiConverterKind,
+    UbiMappingStrategyWire,
+)
+from backend.app.services.agent_judgments_dispatch import (
+    UbiJudgmentGenerationRequest,
+    start_ubi_judgment_generation,
+)
+
+
+class GenerateJudgmentsFromUbiArgs(BaseModel):
+    """Arguments for the ``generate_judgments_from_ubi`` tool.
+
+    Mirrors :class:`backend.app.api.v1.schemas.CreateJudgmentListFromUbiRequest`
+    field-for-field; the same conditional validator + the same
+    dispatcher run both paths.
+    """
+
+    name: str = Field(min_length=1, max_length=256)
+    description: str | None = Field(default=None, max_length=2000)
+    query_set_id: UUID
+    cluster_id: UUID
+    target: str = Field(min_length=1, max_length=256)
+    since: datetime
+    until: datetime | None = None
+    converter: UbiConverterKind
+    converter_config: dict[str, Any] | None = None
+    llm_fill_threshold: int | None = Field(default=20, ge=1)
+    min_impressions_threshold: int | None = Field(default=100, ge=1)
+    mapping_strategy: UbiMappingStrategyWire = "reject"
+    current_template_id: UUID | None = None
+    rubric: str | None = Field(default=None, min_length=1)
+
+    @model_validator(mode="after")
+    def _validate_hybrid_conditional(self) -> GenerateJudgmentsFromUbiArgs:
+        is_hybrid = self.converter == "hybrid_ubi_llm"
+        has_template = self.current_template_id is not None
+        has_rubric = self.rubric is not None
+        if is_hybrid and not (has_template and has_rubric):
+            raise ValueError(
+                "current_template_id and rubric are REQUIRED when converter == 'hybrid_ubi_llm'"
+            )
+        if not is_hybrid and (has_template or has_rubric):
+            raise ValueError(
+                "current_template_id and rubric MUST be null for non-hybrid converters"
+            )
+        return self
+
+
+async def generate_judgments_from_ubi_impl(
+    args: GenerateJudgmentsFromUbiArgs, ctx: ToolContext
+) -> dict[str, Any]:
+    """Start a UBI-derived judgment generation job; return the new judgment_list_id.
+
+    The full preflight (FK resolve, consistency, UBI_NOT_ENABLED probe,
+    window validity + 90-day cap, sync UBI_INSUFFICIENT_DATA gate,
+    hybrid-only LLM preflight, oversize) runs server-side via the
+    shared dispatch helper — same checks the
+    ``POST /api/v1/judgments/generate-from-ubi`` endpoint runs.
+    MUTATING — confirmation required.
+    """
+    result = await start_ubi_judgment_generation(
+        db=ctx.db,
+        redis=ctx.redis,
+        arq_pool=ctx.arq_pool,
+        settings=ctx.settings,
+        req=UbiJudgmentGenerationRequest(
+            name=args.name,
+            description=args.description,
+            query_set_id=str(args.query_set_id),
+            cluster_id=str(args.cluster_id),
+            target=args.target,
+            since=args.since,
+            until=args.until,
+            converter=args.converter,
+            converter_config=args.converter_config,
+            llm_fill_threshold=args.llm_fill_threshold,
+            min_impressions_threshold=args.min_impressions_threshold,
+            mapping_strategy=args.mapping_strategy,
+            current_template_id=(
+                str(args.current_template_id) if args.current_template_id is not None else None
+            ),
+            rubric=args.rubric,
+        ),
+    )
+    return {
+        "judgment_list_id": result.judgment_list_id,
+        "status": result.status,
+    }
+
+
+_DESCRIPTION = (generate_judgments_from_ubi_impl.__doc__ or "").split("\n\n", 1)[0].strip()
+
+GENERATE_JUDGMENTS_FROM_UBI_TOOL: ChatCompletionToolParam = {
+    "type": "function",
+    "function": {
+        "name": "generate_judgments_from_ubi",
+        "description": _DESCRIPTION,
+        "parameters": GenerateJudgmentsFromUbiArgs.model_json_schema(),
+    },
+}

backend/app/api/v1/clusters.py

@@ -66,6 +66,7 @@
     RunQueryRequest,
     RunQueryResponse,
     TargetListResponse,
+    UbiReadinessResponse,
 )
 from backend.app.db import repo
 from backend.app.db.models import Cluster
@@ -90,6 +91,7 @@
     dispatch_run_query,
 )
 from backend.app.services.documents import truncate_source_for_list
+from backend.app.services.ubi_readiness import classify_rung
 
 router = APIRouter()
 logger = structlog.get_logger(__name__)
@@ -337,6 +339,90 @@ async def get_cluster_schema(
 # ---------------------------------------------------------------------------
 
 
+@router.get(
+    "/clusters/{cluster_id}/ubi-readiness",
+    response_model=UbiReadinessResponse,
+    tags=["clusters"],
+)
+async def get_cluster_ubi_readiness(
+    cluster_id: str,
+    query_set_id: Annotated[str, Query(..., min_length=1, max_length=36)],
+    target: Annotated[str, Query(..., min_length=1, max_length=256)],
+    db: Annotated[AsyncSession, Depends(get_db)],
+    redis: Annotated[Redis, Depends(get_redis_client)],
+) -> UbiReadinessResponse:
+    """Classify ``(cluster, query_set, target)`` on the UBI rung ladder.
+
+    feat_ubi_judgments FR-7.
+
+    Required query params: ``query_set_id`` + ``target`` (Spec FR-7 +
+    cycle-3 D-10c: the endpoint MUST 422 without them — the classifier
+    can't compute a per-target rung without an application filter).
+
+    Error envelopes (all per spec §7.5):
+    * ``404 CLUSTER_NOT_FOUND`` — cluster row missing or soft-deleted.
+    * ``404 QUERY_SET_NOT_FOUND`` — query set row missing.
+    * ``422 VALIDATION_ERROR`` — missing required query params (FastAPI's
+      built-in handler, surfaces via ``api/errors.py``).
+    * ``503 CLUSTER_UNREACHABLE`` — adapter cannot reach the cluster.
+
+    The result is cached for 60 s in Redis per
+    ``(cluster_id, query_set_id, target)`` so back-to-back dialog-open
+    and dialog-submit calls don't re-probe.
+    """
+    cluster = await repo.get_cluster(db, cluster_id)
+    if cluster is None:
+        raise _err(404, "CLUSTER_NOT_FOUND", f"cluster {cluster_id} not found", False)
+    query_set = await repo.get_query_set(db, query_set_id)
+    if query_set is None:
+        raise _err(
+            404,
+            "QUERY_SET_NOT_FOUND",
+            f"query set {query_set_id} not found",
+            False,
+        )
+
+    # Consistency: the query set must belong to the requested cluster
+    # (GPT-5.5 PR #317 finding #3, sub-point) — otherwise the rung is
+    # computed against a target the query set was never run on.
+    if query_set.cluster_id != cluster_id:
+        raise _err(
+            422,
+            "VALIDATION_ERROR",
+            f"query_set {query_set_id} belongs to cluster "
+            f"{query_set.cluster_id!r}, not {cluster_id!r}",
+            False,
+        )
+
+    # NOTE: we do NOT pass RelyLoop's internal queries.id values as a
+    # query_id filter. UBI's ubi_events.query_id is the plugin's own UUID,
+    # not queries.id — filtering on the internal ids would match nothing
+    # and silently under-report every cluster to rung_1 (GPT-5.5 PR #317
+    # finding #3). Mapping internal → UBI ids needs the user_query join,
+    # which the readiness probe deliberately skips (it must complete in
+    # <2s per spec §6). The rung is therefore a target-level signal — same
+    # approximation as the dispatcher's U-D2 count.
+    try:
+        async with cluster_svc.acquire_adapter(cluster) as adapter:
+            snapshot = await classify_rung(
+                adapter=adapter,
+                cluster_id=cluster_id,
+                query_set_id=query_set_id,
+                query_set_query_ids=[],
+                target=target,
+                redis=redis,
+            )
+    except (ClusterUnreachable, ClusterUnreachableError) as exc:
+        raise _err(503, "CLUSTER_UNREACHABLE", str(exc), True) from exc
+
+    return UbiReadinessResponse(
+        rung=snapshot.rung,
+        covered_pairs_pct=snapshot.covered_pairs_pct,
+        head_covered=snapshot.head_covered,
+        checked_at=snapshot.checked_at,
+    )
+
+
 @router.get(
     "/clusters/{cluster_id}/targets",
     response_model=TargetListResponse,

backend/app/api/v1/judgments.py

@@ -39,6 +39,7 @@
 from backend.app.api.v1.schemas import (
     CalibrationResponse,
     CalibrationSamplesRequest,
+    CreateJudgmentListFromUbiRequest,
     CreateJudgmentListGenerateRequest,
     GenerateJudgmentsResponse,
     ImportJudgmentListRequest,
@@ -73,7 +74,9 @@
 from backend.app.eval.calibration import compute_calibration
 from backend.app.services.agent_judgments_dispatch import (
     JudgmentGenerationRequest,
+    UbiJudgmentGenerationRequest,
     start_judgment_generation,
+    start_ubi_judgment_generation,
 )
 
 router = APIRouter()
@@ -141,8 +144,10 @@ async def _detail(db: AsyncSession, row: JudgmentList) -> JudgmentListDetail:
         source_breakdown=_SourceBreakdown(
             llm=breakdown.get("llm", 0),
             human=breakdown.get("human", 0),
+            click=breakdown.get("click", 0),
         ),
         calibration=row.calibration,
+        generation_params=row.generation_params,
         created_at=row.created_at,
     )
 
@@ -219,6 +224,76 @@ async def generate_judgments(
                 logger.debug("redis close raised in generate_judgments handler", error=str(exc))
 
 
+# ---------------------------------------------------------------------------
+# POST /api/v1/judgments/generate-from-ubi  (feat_ubi_judgments Story 3.2 / FR-3)
+# ---------------------------------------------------------------------------
+
+
+@router.post(
+    "/judgments/generate-from-ubi",
+    response_model=GenerateJudgmentsResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+    tags=["judgments"],
+)
+async def generate_judgments_from_ubi(
+    body: CreateJudgmentListFromUbiRequest,
+    request: Request,
+    db: Annotated[AsyncSession, Depends(get_db)],
+) -> GenerateJudgmentsResponse:
+    """Start a UBI-derived judgment generation job.
+
+    Delegates to
+    :func:`backend.app.services.agent_judgments_dispatch.start_ubi_judgment_generation`
+    which runs the full FR-4 preflight (U-A..U-H) before INSERT + Arq
+    enqueue. The Pydantic ``model_validator`` on
+    :class:`CreateJudgmentListFromUbiRequest` already enforces the
+    hybrid conditional (``current_template_id`` + ``rubric`` required
+    iff ``converter == 'hybrid_ubi_llm'``); the dispatcher trusts the
+    validated request.
+    """
+    settings = get_settings()
+    arq_pool = getattr(request.app.state, "arq_pool", None)
+    redis_client: Redis | None = None
+
+    try:
+        redis_client = await _open_redis()
+        result = await start_ubi_judgment_generation(
+            db=db,
+            redis=redis_client,
+            arq_pool=arq_pool,
+            settings=settings,
+            req=UbiJudgmentGenerationRequest(
+                name=body.name,
+                description=body.description,
+                query_set_id=body.query_set_id,
+                cluster_id=body.cluster_id,
+                target=body.target,
+                since=body.since,
+                until=body.until,
+                converter=body.converter,
+                converter_config=body.converter_config,
+                llm_fill_threshold=body.llm_fill_threshold,
+                min_impressions_threshold=body.min_impressions_threshold,
+                mapping_strategy=body.mapping_strategy,
+                current_template_id=body.current_template_id,
+                rubric=body.rubric,
+            ),
+        )
+        return GenerateJudgmentsResponse(
+            judgment_list_id=result.judgment_list_id,
+            status=result.status,
+        )
+    finally:
+        if redis_client is not None:
+            try:
+                await redis_client.aclose()
+            except Exception as exc:  # noqa: BLE001 — defensive
+                logger.debug(
+                    "redis close raised in generate_judgments_from_ubi handler",
+                    error=str(exc),
+                )
+
+
 # ---------------------------------------------------------------------------
 # POST /api/v1/judgment-lists/import  (Story 3.2, FR-3b — tutorial path)
 # ---------------------------------------------------------------------------