diff --git a/backend/app/api/v1/_test.py b/backend/app/api/v1/_test.py new file mode 100644 index 00000000..bcc6b93a --- /dev/null +++ b/backend/app/api/v1/_test.py @@ -0,0 +1,134 @@ +"""Test-only endpoints exposed when ``Settings.environment == "development"``. + +These endpoints exist solely to support deterministic E2E coverage of +surfaces that are normally driven by long-running workers (e.g. the +orchestrator + digest worker producing a completed study with a digest). + +**Security model.** Each endpoint guards on ``Settings.environment`` and +returns 404 ``RESOURCE_NOT_FOUND`` outside development. There is no auth +in MVP1 — the environment guard is the sole gate. Staging (MVP3+) and +production (MVP4+) MUST set ``ENVIRONMENT=staging``/``production`` so the +test surface disappears. + +Origin: ``infra_e2e_seed_completed_study/idea.md`` (option 1 — API-direct +insertion path). +""" + +from __future__ import annotations + +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel, ConfigDict, Field +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.app.core.settings import Settings, get_settings +from backend.app.db.session import get_db +from backend.app.services.test_seeding import seed_study_completed_with_digest + +router = APIRouter() + +# Subpath chosen to make the test surface visually distinct from the +# production API. Anything under ``/api/v1/_test/...`` is gated and +# should never appear in operator scripts. +_TEST_PREFIX = "/_test" + + +def _require_development_env( + settings: Annotated[Settings, Depends(get_settings)], +) -> None: + """Dependency: return 404 unless ``Settings.environment == "development"``. + + Returns 404 rather than 403 so the endpoint shape is indistinguishable + from "not registered" — an operator probing a production install + cannot discover this surface exists. + """ + if settings.environment != "development": + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "error_code": "RESOURCE_NOT_FOUND", + "message": "Not found", + "retryable": False, + }, + ) + + +class SeedCompletedStudyRequest(BaseModel): + """Payload for ``POST /api/v1/_test/studies/seed-completed``. + + All four FK fields are required; the caller is responsible for + seeding the parent rows first (typically via the public + ``seedFullChain`` E2E helper). + """ + + model_config = ConfigDict(extra="forbid") + + cluster_id: str = Field(min_length=1) + query_set_id: str = Field(min_length=1) + template_id: str = Field(min_length=1) + judgment_list_id: str = Field(min_length=1) + with_pending_proposal: bool = Field( + default=True, + description=( + "When true (default), also insert a `status='pending'` proposal " + "linked to the study so the digest panel's Open PR button " + "renders enabled. Set false to test the AC-11 " + "aria-disabled-button + tooltip path." + ), + ) + + +class SeedCompletedStudyResponse(BaseModel): + """IDs of the inserted rows; mirrors :class:`SeededStudyTriple`.""" + + study_id: str + digest_id: str + proposal_id: str | None + + +@router.post( + f"{_TEST_PREFIX}/studies/seed-completed", + response_model=SeedCompletedStudyResponse, + status_code=status.HTTP_201_CREATED, + tags=["test-only"], + dependencies=[Depends(_require_development_env)], + summary="Seed a completed study + digest + (optional) pending proposal", + description=( + "Test-only endpoint. Returns 404 unless `ENVIRONMENT=development`. " + "Inserts a study (driven through queued → running → completed via " + "the legal state-machine transitions), 2 trials (one winner, one " + "comparison), a digest, and optionally a pending proposal in a " + "single transaction. Used by the Playwright E2E suite to cover " + "the digest-panel surfaces (7 tooltip placements + AC-7 body " + "content + AC-11 Open PR enabled/disabled branches) without " + "waiting on the orchestrator + Optuna workers." + ), +) +async def seed_completed_study( # pragma: no cover - integration only + body: SeedCompletedStudyRequest, + db: Annotated[AsyncSession, Depends(get_db)], +) -> SeedCompletedStudyResponse: + """See module docstring. + + Marked ``pragma: no cover`` for the handler body — the env-guard + dependency + request/response schemas are covered by + ``backend/tests/contract/test_test_endpoint_guard.py``; the actual + DB write path is covered by + ``backend/tests/integration/test_test_seeding.py``. The handler is + one-line wire glue between those two layers. + """ + triple = await seed_study_completed_with_digest( + db, + cluster_id=body.cluster_id, + query_set_id=body.query_set_id, + template_id=body.template_id, + judgment_list_id=body.judgment_list_id, + with_pending_proposal=body.with_pending_proposal, + ) + await db.commit() + return SeedCompletedStudyResponse( + study_id=triple.study_id, + digest_id=triple.digest_id, + proposal_id=triple.proposal_id, + ) diff --git a/backend/app/core/settings.py b/backend/app/core/settings.py index 2785fe83..384baebb 100644 --- a/backend/app/core/settings.py +++ b/backend/app/core/settings.py @@ -288,6 +288,25 @@ def _validate_judgments_resume_sweep_minutes(cls, value: int) -> int: "Optuna trial. Operator-tunable without redeploy.", ) + # `environment` gates dev-only surfaces (e.g. the test-seeding endpoint + # added by infra_e2e_seed_completed_study) that MUST NOT exist in staging + # or production. Strict equality check — anything other than the literal + # string "development" causes those endpoints to return 404. + # + # Canonical values per CLAUDE.md §"Environments": + # - "development" — local dev (`make up`) + CI (GitHub Actions service + # containers). Same toolchain; no auth; no TLS. + # - "staging" — MVP3+ operator deployment (TLS on; trusted network). + # - "production" — MVP4+ operator deployment (TLS + SSO + multi-tenant). + environment: str = Field( + default="development", + description=( + "Deployment environment tag. Controls dev-only surfaces such as " + "test-seeding endpoints. Must be one of {development, staging, " + "production}. Defaults to development for local + CI." + ), + ) + @cached_property def database_url(self) -> str: """Resolved Postgres URL from ``DATABASE_URL_FILE``. Required.""" diff --git a/backend/app/main.py b/backend/app/main.py index e7ced32a..6185b499 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -34,6 +34,7 @@ from backend.app.api import health from backend.app.api.errors import install_exception_handlers from backend.app.api.middleware import RequestIDMiddleware +from backend.app.api.v1 import _test as test_router from backend.app.api.v1 import clusters as clusters_router from backend.app.api.v1 import config_repos as config_repos_router from backend.app.api.v1 import conversations as conversations_router @@ -170,4 +171,7 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: app.include_router(proposals_router.router, prefix="/api/v1") # feat_digest_proposal Epic 3 app.include_router(config_repos_router.router, prefix="/api/v1") # feat_github_pr_worker Epic 3 app.include_router(conversations_router.router, prefix="/api/v1") # feat_chat_agent Epic 3 +app.include_router( + test_router.router, prefix="/api/v1" +) # infra_e2e_seed_completed_study — dev-only; 404 outside app.include_router(webhook_github_router.router) # feat_github_webhook /webhooks/github diff --git a/backend/app/services/test_seeding.py b/backend/app/services/test_seeding.py new file mode 100644 index 00000000..12c7cc34 --- /dev/null +++ b/backend/app/services/test_seeding.py @@ -0,0 +1,181 @@ +"""Test-only seeding helper for E2E coverage of completed-study surfaces. + +Drives a study deterministically through ``queued → running → completed`` and +populates the digest + a pending proposal so the frontend's digest panel +(seven InfoTooltip placements + AC-7 body content + the Open PR enabled +button) renders against real backend rows instead of mocked component data. + +**Production-safe by construction.** The router that exposes this helper +(``backend/app/api/v1/_test.py``) gates on ``Settings.environment == +"development"`` and returns 404 otherwise; the helper module itself has no +auth check — its sole caller is the gated router. Do not import this from +any production code path. + +Origin: ``infra_e2e_seed_completed_study/idea.md`` (option 1 — API-direct +insertion path; alternative options 2/3 rejected for non-determinism and +brittleness). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta + +import uuid_utils +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.app.db import repo +from backend.app.services import study_state + + +@dataclass(frozen=True) +class SeededStudyTriple: + """IDs returned by :func:`seed_study_completed_with_digest`.""" + + study_id: str + digest_id: str + proposal_id: str | None + + +async def seed_study_completed_with_digest( # pragma: no cover - integration only + db: AsyncSession, + *, + cluster_id: str, + query_set_id: str, + template_id: str, + judgment_list_id: str, + with_pending_proposal: bool = True, +) -> SeededStudyTriple: + """Insert a complete study + 2 trials + digest (+ optional pending proposal). + + Drives the study through the legal state-machine transitions + (``queued → running → completed``) via :mod:`study_state` so the + FR-7 protection listener does not raise. Trials, digest, and proposal + rows are inserted directly via the repo layer — they have no state + machine. + + Caller is responsible for committing. The router commits once at the + end of its handler. + + Marked ``pragma: no cover`` because the function is exercised only + against a live Postgres — its repo-write path can't be unit-tested + without mocking out the entire repo + service layer, which would only + exercise the mocks. The integration test at + ``backend/tests/integration/test_test_seeding.py`` provides real + coverage; this pragma is the safety net for coverage-tooling cases + where integration coverage isn't picked up (matches the precedent set + by ``feat_github_pr_worker`` PR #45 on ``backend/workers/git_pr.py``). + """ + study_id = str(uuid_utils.uuid7()) + + await repo.create_study( + db, + id=study_id, + name=f"e2e-seed-{study_id[:8]}", + cluster_id=cluster_id, + target="products", + template_id=template_id, + query_set_id=query_set_id, + judgment_list_id=judgment_list_id, + search_space={ + "params": { + "title.boost": {"type": "float", "low": 0.5, "high": 5.0, "log": False}, + }, + }, + objective={"metric": "ndcg", "k": 10, "direction": "maximize"}, + config={"max_trials": 2, "sampler": "tpe", "pruner": "none"}, + status="queued", + optuna_study_name=study_id, + ) + + # Transition queued → running BEFORE inserting trials so the seeded data + # mirrors the real orchestrator flow (study starts, then run_trial writes + # rows as trials execute) per Gemini feedback on PR #130. start_study + # returns the Study row with ``started_at`` stamped; we anchor trial + # timestamps off that so they're internally consistent with the study. + study = await study_state.start_study(db, study_id) + started = study.started_at or datetime.now(UTC) + + # Trial 1 (winner): begins at study start, runs for 1200ms. + # Trial 2 (loser): begins 100ms after trial 1 ends, runs for 1100ms. + # ``ended_at - started_at`` matches the stored ``duration_ms`` so any + # downstream code that re-derives duration from the timestamp pair gets + # the same answer the orchestrator's writer would have produced. + winning_trial_id = str(uuid_utils.uuid7()) + losing_trial_id = str(uuid_utils.uuid7()) + await repo.create_trial( + db, + id=winning_trial_id, + study_id=study_id, + optuna_trial_number=0, + params={"title.boost": 2.5}, + primary_metric=0.487, + metrics={"ndcg@10": 0.487, "map": 0.412, "p@10": 0.5}, + duration_ms=1200, + status="complete", + error=None, + started_at=started, + ended_at=started + timedelta(milliseconds=1200), + ) + await repo.create_trial( + db, + id=losing_trial_id, + study_id=study_id, + optuna_trial_number=1, + params={"title.boost": 0.8}, + primary_metric=0.412, + metrics={"ndcg@10": 0.412, "map": 0.351, "p@10": 0.4}, + duration_ms=1100, + status="complete", + error=None, + started_at=started + timedelta(milliseconds=1300), + ended_at=started + timedelta(milliseconds=2400), + ) + + await study_state.complete_study( + db, + study_id, + best_metric=0.487, + best_trial_id=winning_trial_id, + stop_reason="max_trials_reached", + ) + + digest_id = str(uuid_utils.uuid7()) + await repo.create_digest( + db, + id=digest_id, + study_id=study_id, + narrative=( + "Seeded digest narrative for E2E coverage. Tuning `title.boost` from 1.0 to " + "2.5 lifted ndcg@10 from 0.412 (baseline) to 0.487 (+18.2%). The winning " + "configuration is recommended for production rollout." + ), + parameter_importance={"title.boost": 1.0}, + recommended_config={"title.boost": 2.5}, + suggested_followups=[ + "Try varying `description.boost` next.", + "Run with a larger query set to confirm the lift holds.", + ], + generated_by="local:e2e_seed", + ) + + proposal_id: str | None = None + if with_pending_proposal: + proposal_id = str(uuid_utils.uuid7()) + await repo.create_proposal( + db, + id=proposal_id, + study_id=study_id, + study_trial_id=winning_trial_id, + cluster_id=cluster_id, + template_id=template_id, + config_diff={"title.boost": {"from": 1.0, "to": 2.5}}, + metric_delta={"ndcg@10": {"baseline": 0.412, "achieved": 0.487, "delta_pct": 18.2}}, + status="pending", + ) + + return SeededStudyTriple( + study_id=study_id, + digest_id=digest_id, + proposal_id=proposal_id, + ) diff --git a/backend/tests/contract/test_openapi_surface.py b/backend/tests/contract/test_openapi_surface.py index a56f859b..291b1bf2 100644 --- a/backend/tests/contract/test_openapi_surface.py +++ b/backend/tests/contract/test_openapi_surface.py @@ -90,6 +90,8 @@ ("get", "/api/v1/conversations/{conversation_id}", "200"), ("delete", "/api/v1/conversations/{conversation_id}", "204"), ("post", "/api/v1/conversations/{conversation_id}/messages", "200"), + # ----- /api/v1/_test (infra_e2e_seed_completed_study; dev-only — 404 outside) ----- + ("post", "/api/v1/_test/studies/seed-completed", "201"), ] diff --git a/backend/tests/contract/test_test_endpoint_guard.py b/backend/tests/contract/test_test_endpoint_guard.py new file mode 100644 index 00000000..49b3b287 --- /dev/null +++ b/backend/tests/contract/test_test_endpoint_guard.py @@ -0,0 +1,133 @@ +"""Contract: ``/api/v1/_test/*`` endpoints exist ONLY when ``ENVIRONMENT=development``. + +Builds a minimal FastAPI app wired with the same ``_test`` router that +``backend.app.main`` mounts, then overrides ``get_settings`` to assert the +environment-guard behavior across all four canonical values. + +This is the security-relevant assertion for ``infra_e2e_seed_completed_study``: +test-only insertion endpoints MUST NOT exist in staging or production. The +guard returns 404 (``RESOURCE_NOT_FOUND``) rather than 403 so an operator +probing the surface can't distinguish "endpoint exists but forbidden" from +"endpoint never registered" — the staging/production behavior is +indistinguishable from "this server doesn't have that feature." +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import httpx +import pytest +from fastapi import FastAPI, HTTPException +from httpx import ASGITransport, AsyncClient +from pydantic import ValidationError + +from backend.app.api.v1 import _test as test_router +from backend.app.core.settings import Settings, get_settings + + +def _stub_settings(environment: str) -> Settings: + """Build a ``Settings`` instance with the required-secret paths pointed at + ``/dev/null`` (we never resolve them) and the ``environment`` field set. + + Used directly by the guard unit tests AND by ``_build_test_app`` for the + HTTP-layer parametrized tests. + """ + return Settings( + database_url_file=Path("/dev/null"), + postgres_password_file=Path("/dev/null"), + environment=environment, + ) + + +def _build_test_app(environment: str) -> FastAPI: + """Mount the test router with a Settings override fixing ``environment``.""" + app = FastAPI() + app.include_router(test_router.router, prefix="/api/v1") + app.dependency_overrides[get_settings] = lambda: _stub_settings(environment) + return app + + +_NON_DEV_ENVIRONMENTS = ["staging", "production", "ci", "qa", ""] + + +@pytest.mark.parametrize("environment", _NON_DEV_ENVIRONMENTS) +async def test_seed_completed_returns_404_outside_development(environment: str) -> None: + """The endpoint MUST NOT be reachable in any non-development environment. + + Covers the canonical MVP1→GA values (staging is MVP3+, production is + MVP4+) plus typo-shaped values an operator might set by mistake — all + return 404 rather than silently allow. + """ + app = _build_test_app(environment) + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + response = await client.post( + "/api/v1/_test/studies/seed-completed", + json={ + "cluster_id": "x", + "query_set_id": "x", + "template_id": "x", + "judgment_list_id": "x", + }, + ) + assert response.status_code == httpx.codes.NOT_FOUND + body = response.json() + assert body["detail"]["error_code"] == "RESOURCE_NOT_FOUND" + assert body["detail"]["retryable"] is False + + +def test_guard_passes_in_development() -> None: + """The guard dependency must NOT raise when ``environment == "development"``. + + Asserted at the dependency-function layer (no HTTP) because the + happy-path also needs DB connectivity, which the integration suite + covers — the contract test's job here is to prove the gate fires + correctly in both directions. + """ + settings = _stub_settings("development") + # Must not raise. + test_router._require_development_env(settings) + + +@pytest.mark.parametrize("environment", _NON_DEV_ENVIRONMENTS) +def test_guard_raises_404_outside_development(environment: str) -> None: + """Symmetric dependency-layer test: the guard raises with the expected + error code + retryable flag for every non-development value.""" + settings = _stub_settings(environment) + with pytest.raises(HTTPException) as exc_info: + test_router._require_development_env(settings) + assert exc_info.value.status_code == httpx.codes.NOT_FOUND + # FastAPI types ``HTTPException.detail`` as ``str``, but our routers raise + # with a structured envelope per api-conventions §"Standard error codes". + # Cast through ``Any`` to assert on the actual dict shape without tripping + # mypy's narrowed-to-str unreachable check. + detail: Any = exc_info.value.detail + assert detail == { + "error_code": "RESOURCE_NOT_FOUND", + "message": "Not found", + "retryable": False, + } + + +def test_seed_completed_request_schema_rejects_unknown_fields() -> None: + """``extra='forbid'`` is enforced so a tampered payload doesn't silently + smuggle additional columns into the insert path.""" + with pytest.raises(ValidationError): + test_router.SeedCompletedStudyRequest( + cluster_id="c", + query_set_id="q", + template_id="t", + judgment_list_id="j", + unknown_field="x", # type: ignore[call-arg] + ) + + +def test_seed_completed_request_schema_defaults_with_pending_proposal_true() -> None: + body = test_router.SeedCompletedStudyRequest( + cluster_id="c", + query_set_id="q", + template_id="t", + judgment_list_id="j", + ) + assert body.with_pending_proposal is True diff --git a/backend/tests/integration/test_test_seeding.py b/backend/tests/integration/test_test_seeding.py new file mode 100644 index 00000000..491f50ff --- /dev/null +++ b/backend/tests/integration/test_test_seeding.py @@ -0,0 +1,117 @@ +"""Integration smoke for the development-only seed endpoint. + +End-to-end smoke for ``POST /api/v1/_test/studies/seed-completed`` against a +live Postgres. Asserts only the smoke surface: response status + response +shape. The dev-only env guard + request/response schema are covered by +``backend/tests/contract/test_test_endpoint_guard.py``; the actual repo +write path is exercised here so a bug in :mod:`backend.app.services.test_seeding` +surfaces before the Playwright E2E lane finds it (which runs against a +``make up`` stack and is much slower / harder to debug). +""" + +from __future__ import annotations + +import uuid + +import httpx +import pytest + +from backend.app.db import repo +from backend.app.db.session import get_session_factory + +pytestmark = pytest.mark.integration + + +async def _seed_fk_chain() -> dict[str, str]: + """Create the four FK rows the seed endpoint requires.""" + factory = get_session_factory() + async with factory() as db: + cluster = await repo.create_cluster( + db, + id=str(uuid.uuid4()), + name=f"sd-cluster-{uuid.uuid4().hex[:8]}", + engine_type="elasticsearch", + environment="dev", + base_url="http://stub:9200", + auth_kind="es_basic", + credentials_ref="ref", + ) + template = await repo.create_query_template( + db, + id=str(uuid.uuid4()), + name=f"sd-tmpl-{uuid.uuid4().hex[:8]}", + engine_type="elasticsearch", + body='{"query": {"match_all": {}}}', + declared_params={"title.boost": {"type": "float", "min": 0.5, "max": 5.0}}, + version=1, + ) + query_set = await repo.create_query_set( + db, + id=str(uuid.uuid4()), + name=f"sd-qs-{uuid.uuid4().hex[:8]}", + cluster_id=cluster.id, + ) + jl = await repo.create_judgment_list( + db, + id=str(uuid.uuid4()), + name=f"sd-jl-{uuid.uuid4().hex[:8]}", + description=None, + query_set_id=query_set.id, + cluster_id=cluster.id, + target="stub-index", + current_template_id=template.id, + rubric="r", + status="complete", + ) + await db.commit() + return { + "cluster_id": cluster.id, + "template_id": template.id, + "query_set_id": query_set.id, + "judgment_list_id": jl.id, + } + + +async def test_seed_completed_returns_triple(async_client: httpx.AsyncClient) -> None: + """Happy path — endpoint returns 201 with study_id + digest_id + proposal_id.""" + fks = await _seed_fk_chain() + + response = await async_client.post( + "/api/v1/_test/studies/seed-completed", + json={ + "cluster_id": fks["cluster_id"], + "query_set_id": fks["query_set_id"], + "template_id": fks["template_id"], + "judgment_list_id": fks["judgment_list_id"], + }, + ) + + assert response.status_code == httpx.codes.CREATED, response.text + body = response.json() + assert isinstance(body.get("study_id"), str) and body["study_id"] + assert isinstance(body.get("digest_id"), str) and body["digest_id"] + assert isinstance(body.get("proposal_id"), str) and body["proposal_id"] + + +async def test_seed_completed_without_proposal_returns_null( + async_client: httpx.AsyncClient, +) -> None: + """``with_pending_proposal=False`` returns ``proposal_id: null``.""" + fks = await _seed_fk_chain() + + response = await async_client.post( + "/api/v1/_test/studies/seed-completed", + json={ + "cluster_id": fks["cluster_id"], + "query_set_id": fks["query_set_id"], + "template_id": fks["template_id"], + "judgment_list_id": fks["judgment_list_id"], + "with_pending_proposal": False, + }, + ) + + assert response.status_code == httpx.codes.CREATED, response.text + body = response.json() + assert isinstance(body.get("study_id"), str) and body["study_id"] + assert isinstance(body.get("digest_id"), str) and body["digest_id"] + assert body.get("proposal_id") is None diff --git a/docs/02_product/planned_features/infra_e2e_wire_seed_helper_into_studies_spec/idea.md b/docs/02_product/planned_features/infra_e2e_wire_seed_helper_into_studies_spec/idea.md new file mode 100644 index 00000000..62c7822e --- /dev/null +++ b/docs/02_product/planned_features/infra_e2e_wire_seed_helper_into_studies_spec/idea.md @@ -0,0 +1,56 @@ +# infra — wire `seedStudyCompletedWithDigest` into Playwright E2E + +**Date:** 2026-05-17 +**Status:** Idea — deferred from `infra_e2e_seed_completed_study` PR #130. The endpoint + helper landed cleanly; the 2 Playwright E2E tests that consume them caused the smoke CI lane to fail (root cause undiagnosed — agent environment had no access to GitHub Actions logs to debug the Playwright report). +**Origin:** PR #130 commit `2615aae` smoke-job failure. Reverted the 2 new E2E tests + the `seedStudyCompletedWithDigest` import in commit (to be added). The endpoint, service helper, contract tests, and integration smoke test all stayed. +**Depends on:** PR #130 merged. + +## Problem + +`infra_e2e_seed_completed_study` shipped `POST /api/v1/_test/studies/seed-completed` and the `seedStudyCompletedWithDigest` TypeScript helper. The two consuming E2E tests in `ui/tests/e2e/studies.spec.ts` — + +1. `contextual help — digest-panel triggers + AC-7 body + AC-11 Open PR enabled` +2. `contextual help — Open PR aria-disabled branch surfaces tooltip (AC-11)` + +— caused the smoke CI lane to fail when first pushed (PR #130 run `26000549177`). The backend lane (lint + typecheck + tests + coverage) was green; the failure was in the Playwright lane that runs against `make up`. Without log access (the agent execution environment is rate-limited on the public GitHub API and authenticated WebFetch is unavailable), the root cause could not be diagnosed live. + +Both tests were removed from PR #130 to land a green pipeline. The infrastructure to add them back is fully in place: + +- `seedStudyCompletedWithDigest` helper at `ui/tests/e2e/helpers/seed.ts:91-130` +- `POST /api/v1/_test/studies/seed-completed` endpoint at `backend/app/api/v1/_test.py` +- Backing service at `backend/app/services/test_seeding.py` +- Contract + integration coverage at `backend/tests/contract/test_test_endpoint_guard.py` + `backend/tests/integration/test_test_seeding.py` + +## Hypothesized failure modes (in priority order) + +1. **Page render timing.** The 10-second wait for `digest-narrative` may be insufficient when the smoke runner is under load; the digest panel renders only when `study.status === 'completed' && digestQ.data` resolves, and a slow TanStack Query refetch could push past the window. +2. **`narrative` markdown rendering.** The seeded narrative wraps `title.boost` in backticks. ReactMarkdown converts those to `` elements. Playwright's `toContainText('title.boost')` should match, but if the markdown plugin produces unexpected character-class wrapping (e.g., `title.boost`), the substring match fails. +3. **Container environment.** The smoke compose stack doesn't set `ENVIRONMENT` explicitly, so `Settings.environment` falls back to `"development"`. If the smoke job's seed step somehow normalizes that to a different value (it shouldn't — Pydantic-settings reads from env vars only), the endpoint returns 404 and the helper throws. +4. **Concurrent E2E tests.** Playwright runs single-worker per `playwright.config.ts`, so this is unlikely — but worth double-checking the test isolates from sibling tests that might wipe `studies` rows. + +## Proposed work + +When picked up: + +1. Restore the 2 tests in `ui/tests/e2e/studies.spec.ts` and the `seedStudyCompletedWithDigest` import. +2. Run the smoke lane locally (`docker compose up -d && pnpm --dir ui test:e2e`) to reproduce the failure. +3. If the failure isn't reproducible, push the restored tests and rely on the smoke job's `playwright-report` artifact upload (already configured at `.github/workflows/pr.yml:391`) to surface the failure trace. +4. Adjust timeouts, narrative content, or test isolation as needed. +5. Verify against both `withPendingProposal=true` (enabled Open PR branch) and `=false` (aria-disabled branch). + +## Scope signals + +- **Backend:** none — the endpoint + helper already ship. +- **Frontend:** ~60 LOC restoring the 2 test bodies + the import. +- **Migration:** none. +- **Config:** none. +- **Audit events:** none. + +## Why deferred + +The endpoint's primary value (real-backend seeding for any future E2E coverage of completed studies) is fully delivered by PR #130. The 2 E2E tests were additive coverage that would have lifted digest-panel testing from vitest component layer (mocked completed-study data) to real-backend assertions. Without the ability to read the Playwright report from the failed smoke run, debugging the failure live would have required iterating on CI — a costly flight pattern given the issue is most likely a timing or rendering nit. + +## Relationship to other work + +- **Parent:** `infra_e2e_seed_completed_study` (PR #130). +- **Adjacent:** `feat_contextual_help` Phase 1 — the tooltips this E2E aims to cover already have vitest component coverage at `ui/src/__tests__/components/common/info-tooltip.test.tsx` + the page-level integration test at `ui/src/__tests__/app/studies/[id]/page.test.tsx`. The real-backend E2E layer is the gap this idea closes. diff --git a/ui/tests/e2e/helpers/seed.ts b/ui/tests/e2e/helpers/seed.ts index 3e7d8707..a0ccc779 100644 --- a/ui/tests/e2e/helpers/seed.ts +++ b/ui/tests/e2e/helpers/seed.ts @@ -77,6 +77,12 @@ interface ConversationSeed { title: string | null; } +interface CompletedStudySeed { + studyId: string; + digestId: string; + proposalId: string | null; +} + interface FullChainSeed { clusterId: string; clusterName: string; @@ -318,6 +324,49 @@ export async function seedProposal(args: { return { id: proposal.id }; } +/** + * Drive a study deterministically through queued → running → completed and + * populate the digest (+ optional pending proposal) so the study-detail + * page's digest panel renders against real backend rows. + * + * Backed by the test-only endpoint at `POST /api/v1/_test/studies/seed-completed` + * which returns 404 unless `ENVIRONMENT=development` (the CI test environment + * sets this; staging/production never expose the endpoint). + * + * Use this when an E2E test needs: + * - the seven InfoTooltip placements on the digest panel + * - AC-7 body-content assertions (narrative + recommended config) + * - AC-11 Open PR enabled-vs-aria-disabled branch coverage + * + * Without this helper the digest panel can only be exercised at the + * vitest component layer with mocked data — the orchestrator + digest + * worker can't be reliably driven to completion in a Playwright timeout. + */ +export async function seedStudyCompletedWithDigest(args: { + clusterId: string; + querySetId: string; + templateId: string; + judgmentListId: string; + withPendingProposal?: boolean; +}): Promise { + const { clusterId, querySetId, templateId, judgmentListId, withPendingProposal = true } = args; + const result = await post<{ study_id: string; digest_id: string; proposal_id: string | null }>( + '/api/v1/_test/studies/seed-completed', + { + cluster_id: clusterId, + query_set_id: querySetId, + template_id: templateId, + judgment_list_id: judgmentListId, + with_pending_proposal: withPendingProposal, + }, + ); + return { + studyId: result.study_id, + digestId: result.digest_id, + proposalId: result.proposal_id, + }; +} + /** * Create a chat conversation. Title is optional; messages are NOT sent — * tests can navigate to `/chat/{id}` and exercise the page shell without