diff --git a/backend/tests/unit/adapters/test_elastic_render_library.py b/backend/tests/unit/adapters/test_elastic_render_library.py new file mode 100644 index 00000000..8cbdb5da --- /dev/null +++ b/backend/tests/unit/adapters/test_elastic_render_library.py @@ -0,0 +1,256 @@ +# SPDX-FileCopyrightText: 2026 soundminds.ai +# +# SPDX-License-Identifier: Apache-2.0 + +"""Render-validation tests for the ES/OpenSearch library templates +(`chore_template_library_expansion` Story 1.3, FR-6, AC-1). + +For each of the four library templates under ``samples/templates/`` that +target both Elasticsearch 8.11+ and OpenSearch 2.x, this suite: + +1. Loads the template body + the matching ``.search_space.json``. +2. Asserts the search-space keys EQUAL the declared params used by the + render (no extra, no missing) — the platform-equality invariant + enforced by ``backend.app.domain.study.search_space.validate_against_template``. +3. Samples one concrete scalar assignment per parameter (a representative + value, not the ParamSpec dict itself — FR-6 / AC-1). +4. Calls ``ElasticAdapter.render(template, params, query_text)``. +5. Asserts the native block (`multi_match` / `function_score` / `bool` + + `minimum_should_match` / `rescore`) is present in the parsed JSON. + +The same body is then re-rendered as if ``engine_type='opensearch'`` — the +four library shapes are lexical / function-score / rescore DSL that is +identical and valid on both engines (FR-6 explicit engine-agnostic +assertion). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from backend.app.adapters.elastic import ElasticAdapter +from backend.app.adapters.protocol import ParamValue, QueryTemplate +from backend.app.core.settings import get_settings +from backend.app.domain.study.search_space import ( + CategoricalParam, + FloatParam, + IntParam, + SearchSpace, + validate_against_template, +) + +_TEMPLATES_DIR = Path(__file__).resolve().parents[4] / "samples" / "templates" + + +@pytest.fixture(autouse=True) +def _stub_credentials(tmp_path, monkeypatch): + creds = tmp_path / "creds.yaml" + creds.write_text("ref:\n username: u\n password: p\n") + monkeypatch.setenv("DATABASE_URL_FILE", str(tmp_path / "db_url")) + monkeypatch.setenv("POSTGRES_PASSWORD_FILE", str(tmp_path / "pg_pw")) + monkeypatch.setenv("CLUSTER_CREDENTIALS_FILE", str(creds)) + (tmp_path / "db_url").write_text("postgresql+asyncpg://u:p@h/d") + (tmp_path / "pg_pw").write_text("p") + get_settings.cache_clear() + yield + get_settings.cache_clear() + + +def _adapter(engine_type: str = "elasticsearch") -> ElasticAdapter: + return ElasticAdapter( + cluster_id="id", + engine_type=engine_type, # type: ignore[arg-type] + base_url="http://es:9200", + auth_kind="es_basic", + credentials_ref="ref", + engine_config=None, + ) + + +def _sample_assignment(space: SearchSpace) -> dict[str, ParamValue]: + """Pick one concrete scalar per param — first valid value in range / choices. + + Render needs concrete scalars (one float, one int, one categorical + choice), NOT the ParamSpec dict itself. We pick the first choice for + categoricals and the low bound for floats / ints — deterministic so the + test is reproducible. + + Returns a `dict[str, ParamValue]` (the adapter's render() signature + expects `bool | int | float | str | list[str]`). Categorical bool/None + choices are not used in the library, so this works out cleanly. + """ + out: dict[str, ParamValue] = {} + for name, spec in space.params.items(): + if isinstance(spec, FloatParam): + # Mid-range — avoid 0.0/0.5 which can produce JSON quirks. + out[name] = round((spec.low + spec.high) / 2, 3) + elif isinstance(spec, IntParam): + out[name] = spec.low + elif isinstance(spec, CategoricalParam): + choice = spec.choices[0] + # SearchSpace permits bool choices but the library doesn't use them; + # cast keeps mypy happy without runtime conversion. + assert isinstance(choice, (bool, int, float, str)), ( + f"Unexpected categorical choice type {type(choice).__name__}" + ) + out[name] = choice + else: # pragma: no cover — closed pydantic discriminator union today + # Defensive: if a new ParamSpec variant is added to SearchSpace + # in the future, fail loudly rather than silently produce an + # incomplete assignment that surfaces as a hard-to-debug Jinja + # `UndefinedError` downstream. Gemini Code Assist finding on + # PR #416 — accepted. + raise TypeError(f"Unsupported parameter spec type: {type(spec).__name__}") + return out + + +def _load_template_and_space( + name: str, +) -> tuple[QueryTemplate, SearchSpace, dict[str, str]]: # noqa: D401 + """Load the .j2 + .search_space.json; build a QueryTemplate. + + ``declared_params`` is derived from the search-space keys (declared and + search-space must equal exactly per the platform invariant — that is + test-enforced in :mod:`backend.tests.unit.docs.test_template_library_invariants`). + The README registration block is the independent source of truth that + invariant test parses; here we only need *some* declared_params map + that the render's missing-params check is satisfied with. + """ + body = (_TEMPLATES_DIR / f"{name}.j2").read_text() + space_data = json.loads((_TEMPLATES_DIR / f"{name}.search_space.json").read_text()) + space = SearchSpace.model_validate(space_data) + # `spec.type` is a Literal but `declared_params` accepts plain `str`; + # widen the value type with `str(...)` so the dict invariance dance + # mypy demands doesn't bleed into every caller. + declared_params: dict[str, str] = {key: str(spec.type) for key, spec in space.params.items()} + template = QueryTemplate( + name=name, + engine_type="elasticsearch", + body=body, + declared_params=declared_params, + ) + # Sanity: equality holds. If this fires, the .search_space.json drifted + # from the template's intent. + validate_against_template(space, declared_params, name) + return template, space, declared_params + + +# --------------------------------------------------------------------------- +# Per-template render cases +# --------------------------------------------------------------------------- + + +class TestMultiMatchBasic: + def test_renders_to_native_multi_match(self) -> None: + template, space, _ = _load_template_and_space("multi_match_basic") + params = _sample_assignment(space) + native = _adapter().render(template, params=params, query_text="laptop") + assert native.query_id == "multi_match_basic" + block = native.body["query"]["multi_match"] + assert block["type"] == "best_fields" + assert block["query"] == "laptop" + assert "tie_breaker" in block + # Engine-agnostic structure: no ES-only keys (`retriever`, `rrf`) leak in. + assert "retriever" not in native.body + assert "rrf" not in native.body + + def test_renders_identically_on_opensearch(self) -> None: + template, space, _ = _load_template_and_space("multi_match_basic") + params = _sample_assignment(space) + es_body = _adapter("elasticsearch").render(template, params, "laptop").body + os_body = _adapter("opensearch").render(template, params, "laptop").body + assert es_body == os_body # lexical DSL — byte-identical across engines + + +class TestFunctionScoreDecay: + def test_renders_to_function_score_with_gauss(self) -> None: + template, space, _ = _load_template_and_space("function_score_decay") + params = _sample_assignment(space) + native = _adapter().render(template, params=params, query_text="phone") + block = native.body["query"]["function_score"] + assert "functions" in block + assert block["functions"][0]["gauss"]["created_at"] + assert block["boost_mode"] == "multiply" + # The inner lexical pass is best_fields lexical (engine-agnostic). + assert block["query"]["multi_match"]["type"] == "best_fields" + + +class TestBoolBoosted: + def test_renders_to_bool_with_min_should_match(self) -> None: + template, space, _ = _load_template_and_space("bool_boosted") + params = _sample_assignment(space) + native = _adapter().render(template, params=params, query_text="shoes") + block = native.body["query"]["bool"] + assert "must" in block + assert "should" in block + # FR-1 names the must/should/filter shape — the filter clause is a + # baked-in `exists` floor on `title` (GPT-5.5 cycle-3 finding). + assert "filter" in block + assert block["filter"][0]["exists"]["field"] == "title" + assert "minimum_should_match" in block + # All three field boosts wired through to the should clauses. + should_fields = {list(c["match"].keys())[0] for c in block["should"]} + assert should_fields == {"title", "description", "bullet_points"} + + +class TestRescorePhrase: + def test_renders_with_rescore_block(self) -> None: + template, space, _ = _load_template_and_space("rescore_phrase") + params = _sample_assignment(space) + native = _adapter().render(template, params=params, query_text="leather sofa") + # First pass: best_fields lexical. + assert native.body["query"]["multi_match"]["type"] == "best_fields" + # Second pass: phrase rescore over the title field. + rescore = native.body["rescore"] + assert "window_size" in rescore + assert rescore["query"]["rescore_query"]["match_phrase"]["title"]["query"] == "leather sofa" + # phrase_slop is the canonical knob — confirm it landed in the phrase block. + assert "slop" in rescore["query"]["rescore_query"]["match_phrase"]["title"] + + +# --------------------------------------------------------------------------- +# Engine-agnostic sweep — one assertion documenting that the 4 ES/OS library +# templates render identically for ES and OpenSearch (lexical / function-score +# / rescore DSL is shared between the two engines). +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "template_name", + ["multi_match_basic", "function_score_decay", "bool_boosted", "rescore_phrase"], +) +def test_es_and_opensearch_bodies_are_identical(template_name: str) -> None: + template, space, _ = _load_template_and_space(template_name) + params = _sample_assignment(space) + es_body = _adapter("elasticsearch").render(template, params, "widget").body + os_body = _adapter("opensearch").render(template, params, "widget").body + assert es_body == os_body, f"{template_name} diverged between ES and OpenSearch" + + +# --------------------------------------------------------------------------- +# JSON-safety sweep — a query_text containing a double-quote, backslash, or +# newline must still render valid JSON (the templates wrap query_text via the +# Jinja `tojson` filter). GPT-5.5 cycle-3 finding — accepted: a naive +# `"{{ query_text }}"` would break `json.loads` on such input. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "template_name", + ["multi_match_basic", "function_score_decay", "bool_boosted", "rescore_phrase"], +) +def test_renders_valid_json_for_query_with_special_chars(template_name: str) -> None: + template, space, _ = _load_template_and_space(template_name) + params = _sample_assignment(space) + # Double-quote + backslash + newline — the canonical JSON-breaking trio. + nasty = 'a "quoted" \\ value\nwith newline' + native = _adapter().render(template, params=params, query_text=nasty) + # render() already json.loads-es internally; reaching here means valid JSON. + # Confirm the query text round-trips intact somewhere in the body. + body_str = json.dumps(native.body) + assert "quoted" in body_str + # The literal newline survived as an escaped \n in the parsed structure. + assert "with newline" in body_str diff --git a/backend/tests/unit/adapters/test_solr_render.py b/backend/tests/unit/adapters/test_solr_render.py index f5ad1818..0a0390bf 100644 --- a/backend/tests/unit/adapters/test_solr_render.py +++ b/backend/tests/unit/adapters/test_solr_render.py @@ -153,6 +153,95 @@ def test_lucene_template_renders(self, adapter) -> None: assert "tie" not in nq.body +# --------------------------------------------------------------------------- +# Library templates (chore_template_library_expansion Story 1.3, FR-6) — the +# Solr ``edismax_basic.j2`` and ``boost_decay.j2`` library templates live +# directly under ``samples/templates/solr/`` (not under ``products_*`` like +# the demo trio) and pair with a checked-in ``.search_space.json``. +# --------------------------------------------------------------------------- + + +def _solr_library_template(name: str, declared_params: dict[str, str]) -> QueryTemplate: + """Like ``_solr_template`` but reads ``samples/templates/solr/.j2`` + rather than the ``products_.j2`` demo path.""" + body = (Path(__file__).resolve().parents[4] / f"samples/templates/solr/{name}.j2").read_text() + return QueryTemplate( + name=name, + engine_type="solr", + body=body, + declared_params=declared_params, + ) + + +class TestSolrLibraryTemplates: + def test_edismax_basic_renders(self, adapter) -> None: + tpl = _solr_library_template( + "edismax_basic", + declared_params={ + "title_boost": "float", + "description_boost": "categorical", + "bullet_points_boost": "categorical", + "tie": "categorical", + "mm": "categorical", + "ps": "int", + }, + ) + nq = adapter.render( + tpl, + params={ + "title_boost": 2.0, + "description_boost": 1.0, + "bullet_points_boost": 0.5, + "tie": 0.3, + "mm": "75%", + "ps": 2, + }, + query_text="laptop", + ) + assert nq.body["defType"] == "edismax" + assert nq.body["q"] == "laptop" + # field_boosts → qf (post-pivot, space-joined, source-order preserved). + assert nq.body["qf"] == "title^2.0 description^1.0 bullet_points^0.5" + # `pf` is baked in (literal) so the declared-tunable `ps` (phrase slop) + # actually has phrase queries to act on (spec FR-2 / Gemini fix). + assert nq.body["pf"] == "title description" + assert nq.body["tie"] == "0.3" + assert nq.body["mm"] == "75%" + # slop → ps pivot. + assert nq.body["ps"] == "2" + assert nq.body["fl"] == "*,score" + + def test_boost_decay_renders_with_bf(self, adapter) -> None: + tpl = _solr_library_template( + "boost_decay", + declared_params={ + "title_boost": "float", + "description_boost": "float", + "bullet_points_boost": "categorical", + "boost_weight": "categorical", + "decay_scale": "categorical", + }, + ) + nq = adapter.render( + tpl, + params={ + "title_boost": 2.0, + "description_boost": 1.0, + "bullet_points_boost": 0.5, + "boost_weight": 1.0, + "decay_scale": "3e-11", + }, + query_text="laptop", + ) + assert nq.body["defType"] == "edismax" + # field_boosts → qf. + assert nq.body["qf"] == "title^2.0 description^1.0 bullet_points^0.5" + # The bf string scales a 0→1 recip() decay curve by boost_weight via + # product(...) so the max additive boost (at age 0) equals boost_weight. + # m = decay_scale (string); recip numerator/denominator are fixed at 1. + assert nq.body["bf"] == "product(1.0,recip(ms(NOW,created_at),3e-11,1,1))" + + # --------------------------------------------------------------------------- # Pivot helpers — individual coverage so a broken pivot surfaces in isolation. # --------------------------------------------------------------------------- diff --git a/backend/tests/unit/docs/test_template_library_invariants.py b/backend/tests/unit/docs/test_template_library_invariants.py new file mode 100644 index 00000000..631e4610 --- /dev/null +++ b/backend/tests/unit/docs/test_template_library_invariants.py @@ -0,0 +1,268 @@ +# SPDX-FileCopyrightText: 2026 soundminds.ai +# +# SPDX-License-Identifier: Apache-2.0 + +"""Doc-consistency invariants for the runnable template library +(``chore_template_library_expansion`` Story 1.3, Epic 1). + +These tests do NOT depend on the per-engine tunable-params cheatsheets — +those land in Epic 2 and are exercised by +``test_tunable_params_cheatsheets.py``. The Epic-1 invariants are: + +1. For each runnable library template: + - **Parse** the README registration block (in + ``samples/templates/README.md`` for ES/OS templates, in + ``samples/templates/solr/README.md`` for Solr templates) to extract + its ``declared_params`` keys — independently from the + ``.search_space.json`` source. A bad README block fails the test. + - Assert those parsed keys EQUAL the keys in the corresponding + ``.search_space.json``. (Platform-equality invariant per + ``backend/app/domain/study/search_space.py:validate_against_template``.) + - Assert the ``.search_space.json`` cardinality is ≤ 10⁶ using the + same ``SearchSpace.estimate_cardinality`` the study builder uses. + +2. Each ES/OpenSearch template's registration block MUST be + parameterized via ``ENGINE_TYPE="elasticsearch" # or opensearch`` (the + same body is engine-agnostic; the operator picks the engine per + registration). Solr blocks MUST hard-code ``engine_type: "solr"``. + +3. The four existing demo templates (``product_search.j2`` + + ``solr/products_*.j2``) MUST remain byte-stable — AC-3. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +from backend.app.domain.study.search_space import SearchSpace, estimate_cardinality + +REPO_ROOT = Path(__file__).resolve().parents[4] +TEMPLATES_DIR = REPO_ROOT / "samples" / "templates" +SAMPLES_README = TEMPLATES_DIR / "README.md" +SOLR_README = TEMPLATES_DIR / "solr" / "README.md" + + +# Locked list of runnable library templates (spec FR-1 + FR-2). Listed here +# rather than discovered from disk so a missing template fails loudly +# instead of being silently dropped. +ES_OS_TEMPLATES = [ + "multi_match_basic", + "function_score_decay", + "bool_boosted", + "rescore_phrase", +] +SOLR_TEMPLATES = [ + "edismax_basic", + "boost_decay", +] + + +# --------------------------------------------------------------------------- +# README registration-block parser +# --------------------------------------------------------------------------- + + +def _extract_block(readme_text: str, template_filename: str) -> str: + r"""Return the README section for ````. + + Sections begin with ``### \`\``` and end at the next + ``### \`...\`` heading OR at the next ``## ``-level heading OR end-of-file. + """ + marker = f"### `{template_filename}`" + start = readme_text.find(marker) + if start == -1: + raise AssertionError( + f"README registration block not found for `{template_filename}`. " + "Each runnable library template MUST have a section heading " + f"`{marker}` in its samples README." + ) + # Find the next section / chapter heading after `start + len(marker)`. + next_h3 = readme_text.find("\n### ", start + len(marker)) + next_h2 = readme_text.find("\n## ", start + len(marker)) + candidates = [pos for pos in (next_h3, next_h2) if pos != -1] + end = min(candidates) if candidates else len(readme_text) + return readme_text[start:end] + + +def _parse_declared_params_block(section: str) -> dict[str, str]: + """Extract the ``declared_params: { ... }`` portion of the jq command. + + Robust against newlines and indentation; matches ``: ""`` + pairs inside the dict. The trailing pair has no comma — the regex + captures both shapes. + """ + # Find the substring after `declared_params:` up to the matching `}`. + m = re.search(r"declared_params:\s*\{([^{}]*)\}", section) + if m is None: + raise AssertionError( + "Could not locate `declared_params: { ... }` inside the README " + "registration block. Each runnable template's curl block MUST " + "include a `declared_params` map (spec FR-3)." + ) + body = m.group(1) + # Accept both shapes: jq-style unquoted keys (`title_boost: "float"`) and + # standard-JSON quoted keys (`"title_boost": "float"`). Gemini Code Assist + # finding on PR #416 — accepted: a future operator who copies a raw-JSON + # block in would otherwise silently fail with zero pairs extracted. + pairs = dict(re.findall(r'"?(\w+)"?:\s*"(\w+)"', body)) + if not pairs: + raise AssertionError( + '`declared_params` block parsed but no `: ""` pairs ' + f"were extracted. Block content: {body!r}" + ) + return pairs + + +def _extract_engine_type(section: str) -> str: + """Return the engine_type string passed to the registration call. + + For ES/OS templates this is the parameterized form (the section sets + ``ENGINE_TYPE="elasticsearch" # or opensearch`` and the jq command + threads it through ``--arg engine "$ENGINE_TYPE"`` + ``engine_type: $engine``). + For Solr templates the value is a literal ``"solr"`` string. The return + value lets the caller distinguish the two shapes. + """ + # Hard-coded literal: `engine_type: ""`. + literal = re.search(r'engine_type:\s*"(\w+)"', section) + if literal: + return literal.group(1) + # Parameterized via $engine binding. + if "engine_type: $engine" in section: + return "$engine" + raise AssertionError( + "Registration block must set `engine_type` either to a literal " + '(e.g. `engine_type: "solr"`) or to `engine_type: $engine` paired ' + 'with a `--arg engine "$ENGINE_TYPE"` shell-variable invocation. ' + "Neither shape was found." + ) + + +def _load_search_space(template: str, solr: bool = False) -> SearchSpace: + subdir = TEMPLATES_DIR / "solr" if solr else TEMPLATES_DIR + raw = json.loads((subdir / f"{template}.search_space.json").read_text()) + return SearchSpace.model_validate(raw) + + +# --------------------------------------------------------------------------- +# Per-template invariants +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("template_name", ES_OS_TEMPLATES) +def test_es_os_template_declared_params_match_search_space(template_name: str) -> None: + section = _extract_block(SAMPLES_README.read_text(), f"{template_name}.j2") + readme_keys = set(_parse_declared_params_block(section).keys()) + space = _load_search_space(template_name) + space_keys = set(space.params.keys()) + assert readme_keys == space_keys, ( + f"README declared_params keys for `{template_name}.j2` diverged from " + f"its .search_space.json keys.\n" + f" README only: {sorted(readme_keys - space_keys)}\n" + f" search-space only: {sorted(space_keys - readme_keys)}\n" + "Edit the README registration block OR the .search_space.json so " + "they EQUAL exactly — the platform validator " + "`validate_against_template` rejects any drift at runtime." + ) + + +@pytest.mark.parametrize("template_name", SOLR_TEMPLATES) +def test_solr_template_declared_params_match_search_space(template_name: str) -> None: + section = _extract_block(SOLR_README.read_text(), f"{template_name}.j2") + readme_keys = set(_parse_declared_params_block(section).keys()) + space = _load_search_space(template_name, solr=True) + space_keys = set(space.params.keys()) + assert readme_keys == space_keys, ( + f"README declared_params keys for `solr/{template_name}.j2` diverged " + f"from its .search_space.json keys.\n" + f" README only: {sorted(readme_keys - space_keys)}\n" + f" search-space only: {sorted(space_keys - readme_keys)}" + ) + + +@pytest.mark.parametrize("template_name", ES_OS_TEMPLATES + SOLR_TEMPLATES) +def test_search_space_cardinality_at_or_below_cap(template_name: str) -> None: + is_solr = template_name in SOLR_TEMPLATES + space = _load_search_space(template_name, solr=is_solr) + cardinality = estimate_cardinality(space) + # Spec §9 + FR-3 require starter spaces to stay STRICTLY under 10⁶ + # (the platform's own SearchSpace validator rejects > 10⁶, but the + # library contract is the tighter `<`). GPT-5.5 final-review cycle-3 + # finding — accepted: an exactly-10⁶ space is at the platform ceiling + # and leaves no headroom. + assert cardinality < 1_000_000, ( + f"`{template_name}.search_space.json` cardinality {cardinality} is not < 10^6. " + "Narrow ranges, drop a float to categorical, or shrink categorical " + "choice sets so trial counts stay tractable." + ) + + +# --------------------------------------------------------------------------- +# Engine-type parameterization (cycle 4, GPT-5.5 F1) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("template_name", ES_OS_TEMPLATES) +def test_es_os_registration_block_is_engine_parameterized(template_name: str) -> None: + """Each ES/OS template's curl block MUST be parameterized via + ``ENGINE_TYPE="elasticsearch" # or opensearch`` because the bodies are + engine-agnostic but `query_templates.engine_type` is single-valued + per row — operators register the same body once per engine they run. + """ + section = _extract_block(SAMPLES_README.read_text(), f"{template_name}.j2") + assert _extract_engine_type(section) == "$engine", ( + f"ES/OS template `{template_name}.j2`'s registration block must thread " + 'engine_type through `$engine`. Set `ENGINE_TYPE="elasticsearch" # ' + 'or opensearch` above the jq command and use `--arg engine "$ENGINE_TYPE"`.' + ) + # The shell variable + the comment hint must both be present. + assert 'ENGINE_TYPE="elasticsearch"' in section, ( + f"ES/OS template `{template_name}.j2`'s block is missing the literal " + '`ENGINE_TYPE="elasticsearch"` initializer.' + ) + assert "# or opensearch" in section, ( + f"ES/OS template `{template_name}.j2`'s block must annotate that " + "the same body is also valid for OpenSearch with a `# or opensearch` comment." + ) + + +@pytest.mark.parametrize("template_name", SOLR_TEMPLATES) +def test_solr_registration_block_uses_literal_solr_engine(template_name: str) -> None: + section = _extract_block(SOLR_README.read_text(), f"{template_name}.j2") + assert _extract_engine_type(section) == "solr", ( + f"Solr template `solr/{template_name}.j2`'s registration block must " + 'set `engine_type: "solr"` (Solr templates are not engine-agnostic).' + ) + + +# --------------------------------------------------------------------------- +# AC-3 — four existing demo templates are byte-identical to `main`. +# --------------------------------------------------------------------------- + + +# Spec AC-3 cites the four demo template paths plus the demo's reader at +# `demo_seeding.py:1248`. We assert the files exist + carry their expected +# *signatures* — header-line markers chosen to be stable identifiers that +# change only if the body is rewritten. If you intentionally edit any of +# these, update this test in lockstep (and bump the demo reseed verification). +DEMO_TEMPLATE_SIGNATURES = { + TEMPLATES_DIR / "product_search.j2": "product_search.j2 — canonical demo Jinja2", + TEMPLATES_DIR + / "solr" + / "products_edismax.j2": "products_edismax.j2 — canonical Apache Solr edismax", + TEMPLATES_DIR / "solr" / "products_dismax.j2": "products_dismax.j2", + TEMPLATES_DIR / "solr" / "products_lucene.j2": "products_lucene.j2", +} + + +@pytest.mark.parametrize("path,signature", list(DEMO_TEMPLATE_SIGNATURES.items())) +def test_demo_template_unchanged(path: Path, signature: str) -> None: + assert path.is_file(), f"Demo template {path} was removed — AC-3 violation" + assert signature in path.read_text(), ( + f"Demo template {path} header signature changed — `{signature}` no " + "longer appears. AC-3 protects these files (demo_seeding + smoke depend " + "on them); this chore must not touch them." + ) diff --git a/backend/tests/unit/docs/test_tunable_params_cheatsheets.py b/backend/tests/unit/docs/test_tunable_params_cheatsheets.py new file mode 100644 index 00000000..326aeb04 --- /dev/null +++ b/backend/tests/unit/docs/test_tunable_params_cheatsheets.py @@ -0,0 +1,326 @@ +# SPDX-FileCopyrightText: 2026 soundminds.ai +# +# SPDX-License-Identifier: Apache-2.0 + +"""Doc-consistency invariants for the per-engine tunable-params cheatsheets +(``chore_template_library_expansion`` Story 2.4, Epic 2 — AC-4, AC-1b). + +These tests assert against files created in Epic 2 (`elasticsearch-`, +`opensearch-`, `solr-tunable-params.md`) so they live in their own module +— the Epic-1 invariants in ``test_template_library_invariants.py`` don't +depend on cheatsheet content. + +Coverage: + +1. **Required-knob inventory per cheatsheet (AC-4):** each cheatsheet + covers the 8 unified params from + ``docs/01_architecture/adapters.md`` PLUS every declared param + exposed by that engine's runnable templates. A missing knob fails + the test loudly. + +2. **"Templates that use this param" back-links resolve:** each + back-link in a cheatsheet names a template that actually declares + that param. + +3. **Vendor-docs README index has a row per cheatsheet.** + +4. **FR-1b kNN + hybrid snippets parse as JSON** (ES + OpenSearch + cheatsheets only — Solr ships no vector snippet by design). + +5. **OpenSearch hybrid uses the normalization-processor construct, + not the ES `rrf` retriever** (FR-1b — the two engines diverge here). +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[4] +VENDOR_DOCS = REPO_ROOT / "docs" / "06_vendor_docs" +TEMPLATES_DIR = REPO_ROOT / "samples" / "templates" + +ES_CHEATSHEET = VENDOR_DOCS / "elasticsearch-tunable-params.md" +OS_CHEATSHEET = VENDOR_DOCS / "opensearch-tunable-params.md" +SOLR_CHEATSHEET = VENDOR_DOCS / "solr-tunable-params.md" +VENDOR_README = VENDOR_DOCS / "README.md" + + +# The 8 unified params from ``docs/01_architecture/adapters.md`` +# "Cross-engine parameter naming". Source-of-truth comment: +# // Values must match docs/01_architecture/adapters.md §"Cross-engine parameter naming" +UNIFIED_PARAMS = [ + "field_boosts", + "phrase_field_boosts", + "tie_breaker", + "min_should_match", + "fuzziness", + "slop", + "boost_fn", + "rerank_model", +] + +# Per-engine declared params from the runnable library templates. Each +# of these names MUST appear (substring match) somewhere in the matching +# cheatsheet — either as its own section or in a "Templates that use +# this param" back-link. +ES_OS_TEMPLATE_PARAMS = { + "multi_match_basic": [ + "title_boost", + "description_boost", + "bullet_points_boost", + "tie_breaker", + "fuzziness", + ], + "function_score_decay": [ + "title_boost", + "description_boost", + "bullet_points_boost", + "decay_scale", + "decay_offset", + "decay_decay", + ], + "bool_boosted": ["title_boost", "description_boost", "bullet_points_boost", "min_should_match"], + "rescore_phrase": [ + "title_boost", + "description_boost", + "bullet_points_boost", + "rescore_window_size", + "rescore_query_weight", + "rescore_phrase_slop", + ], +} +SOLR_TEMPLATE_PARAMS = { + "edismax_basic": ["title_boost", "description_boost", "bullet_points_boost", "tie", "mm", "ps"], + "boost_decay": [ + "title_boost", + "description_boost", + "bullet_points_boost", + "boost_weight", + "decay_scale", + ], +} + + +# --------------------------------------------------------------------------- +# Required-knob inventory (AC-4) +# --------------------------------------------------------------------------- + + +def _flatten(params: dict[str, list[str]]) -> set[str]: + out: set[str] = set() + for vals in params.values(): + out.update(vals) + return out + + +def _section_headings(text: str) -> list[str]: + """Return the ``### `` ...`` heading slugs in document order. + + Splits on the first comma inside the heading so a grouped header like + ``### `decay_scale`, `decay_offset`, `decay_decay``` registers all + three slugs (the cheatsheets group the three decay params under a + single heading per the cheatsheet design). + """ + out: list[str] = [] + for line in text.splitlines(): + if not line.startswith("### "): + continue + # Extract every ``…`` token from the heading (handles grouped + # headings like the decay-trio). + for m in re.findall(r"`(\w+)`", line): + out.append(m) + return out + + +@pytest.mark.parametrize( + "cheatsheet,engine_params", + [ + (ES_CHEATSHEET, _flatten(ES_OS_TEMPLATE_PARAMS)), + (OS_CHEATSHEET, _flatten(ES_OS_TEMPLATE_PARAMS)), + (SOLR_CHEATSHEET, _flatten(SOLR_TEMPLATE_PARAMS)), + ], + ids=["elasticsearch", "opensearch", "solr"], +) +def test_cheatsheet_covers_all_required_knobs(cheatsheet: Path, engine_params: set[str]) -> None: + """AC-4: every cheatsheet covers (a) the 8 unified params (each as its + own ``### ```` section heading per the plan's AC-4 wording) and + (b) every declared param exposed by that engine's runnable templates. + + Per-template-instance declared params (`title_boost`, `description_boost`, + `bullet_points_boost`) are intentionally grouped under the unified + `field_boosts` section rather than promoted to their own headings — + the cheatsheet design documents the CONCEPT once and lists template + instances in the back-link line. The substring check covers them. + """ + text = cheatsheet.read_text() + headings = set(_section_headings(text)) + + # Strict heading check for the 8 unified params (GPT-5.5 final-review + # finding on PR #416 — accepted: substring-only was weaker than the + # plan's "section/anchor for all 8 unified params" wording). + missing_unified = sorted(p for p in UNIFIED_PARAMS if p not in headings) + assert not missing_unified, ( + f"{cheatsheet.name} is missing dedicated section headings (### ``...) " + f"for required unified params: {missing_unified}. Each of the 8 unified " + "params in `docs/01_architecture/adapters.md` MUST have its own section " + "heading in every per-engine cheatsheet." + ) + + # Substring check for declared-param instances + ES-specific knobs + # (these may appear as their own heading OR in a back-link line within + # a unified-vocabulary section). + declared_only = engine_params - set(UNIFIED_PARAMS) + missing_declared = sorted(p for p in declared_only if p not in text) + assert not missing_declared, ( + f"{cheatsheet.name} is missing entries for declared params: {missing_declared}. " + "Each declared param must appear somewhere in the cheatsheet — either " + "as its own section heading or in a per-knob 'Templates that use " + "this param' back-link." + ) + + +# --------------------------------------------------------------------------- +# Back-links resolve +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "cheatsheet,engine_templates", + [ + (ES_CHEATSHEET, list(ES_OS_TEMPLATE_PARAMS.keys())), + (OS_CHEATSHEET, list(ES_OS_TEMPLATE_PARAMS.keys())), + (SOLR_CHEATSHEET, list(SOLR_TEMPLATE_PARAMS.keys())), + ], + ids=["elasticsearch", "opensearch", "solr"], +) +def test_cheatsheet_backlinks_name_real_templates( + cheatsheet: Path, engine_templates: list[str] +) -> None: + """Every cheatsheet that mentions a `.j2` back-link must + name a template that actually exists in the runnable library.""" + text = cheatsheet.read_text() + # Find `.j2` references (excluding the demo `products_*.j2` which + # the cheatsheets may reference for context — those exist on disk too). + referenced = set(re.findall(r"`(\w+)\.j2`", text)) + # Allow demo templates + any locked engine_templates name. + valid_names = set(engine_templates) | { + "product_search", + "products_edismax", + "products_dismax", + "products_lucene", + } + unknown = referenced - valid_names + assert not unknown, ( + f"{cheatsheet.name} references unknown templates: {sorted(unknown)}. " + "Each `.j2` reference must point at a real runnable library " + f"template ({sorted(valid_names)}) or a demo template." + ) + + +# --------------------------------------------------------------------------- +# Vendor README index rows +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "cheatsheet_filename", + [ + "elasticsearch-tunable-params.md", + "opensearch-tunable-params.md", + "solr-tunable-params.md", + ], +) +def test_vendor_readme_has_index_row(cheatsheet_filename: str) -> None: + text = VENDOR_README.read_text() + assert f"`{cheatsheet_filename}`" in text or f"({cheatsheet_filename})" in text, ( + f"Vendor-docs README missing an index row for `{cheatsheet_filename}`. " + "Add a row to the index table per FR-5." + ) + + +# --------------------------------------------------------------------------- +# FR-1b — vector / hybrid reference snippets parse as JSON +# --------------------------------------------------------------------------- + + +def _extract_json_blocks(markdown: str) -> list[str]: + """Return the contents of every fenced ```json``` block in the file.""" + return re.findall(r"```json\n(.*?)```", markdown, flags=re.DOTALL) + + +@pytest.mark.parametrize("cheatsheet", [ES_CHEATSHEET, OS_CHEATSHEET], ids=["es", "os"]) +def test_es_os_cheatsheet_json_snippets_parse(cheatsheet: Path) -> None: + """FR-1b: the kNN + hybrid reference snippets MUST be valid JSON. + Placeholder values like `""` are + quoted strings — they parse fine; the test catches typos in braces, + commas, and key ordering.""" + blocks = _extract_json_blocks(cheatsheet.read_text()) + assert blocks, f"{cheatsheet.name} has no fenced JSON blocks — FR-1b expects ≥ 2" + for i, block in enumerate(blocks): + try: + json.loads(block) + except json.JSONDecodeError as exc: + pytest.fail( + f"{cheatsheet.name} JSON block #{i + 1} does not parse: {exc.msg}\n" + f"Block content:\n{block}" + ) + + +# --------------------------------------------------------------------------- +# FR-1b — OpenSearch hybrid uses normalization-processor, not `rrf` +# --------------------------------------------------------------------------- + + +def test_opensearch_hybrid_does_not_use_rrf_retriever() -> None: + text = OS_CHEATSHEET.read_text() + # The OpenSearch cheatsheet may MENTION the ES rrf retriever (to call + # out the divergence), but the OS hybrid SNIPPET must NOT use it. + # Look at fenced JSON blocks specifically — those are the operator- + # facing snippets. + blocks = _extract_json_blocks(text) + for i, block in enumerate(blocks): + assert '"rrf"' not in block, ( + f"opensearch-tunable-params.md JSON block #{i + 1} contains an " + "`rrf` retriever — that's an Elasticsearch-only construct. " + "Use the OpenSearch search-pipeline normalization processor." + ) + # Positive assertion: the OS cheatsheet should explicitly mention the + # normalization-processor construct (FR-1b requirement). + assert "normalization" in text.lower() and "processor" in text.lower(), ( + "opensearch-tunable-params.md must document the normalization-processor construct (FR-1b)." + ) + + +def test_elasticsearch_hybrid_uses_rrf_retriever() -> None: + """FR-1b: the ES cheatsheet's hybrid section MUST use the `rrf` + retriever (8.11+ native construct), not the OpenSearch + normalization-processor.""" + text = ES_CHEATSHEET.read_text() + blocks = _extract_json_blocks(text) + found_rrf = any('"rrf"' in block for block in blocks) + assert found_rrf, ( + "elasticsearch-tunable-params.md must include an `rrf` retriever " + "snippet (FR-1b) demonstrating the ES-native hybrid construct." + ) + + +# --------------------------------------------------------------------------- +# FR-5 — samples READMEs link to the cheatsheets +# --------------------------------------------------------------------------- + + +def test_samples_readme_links_cheatsheets() -> None: + text = (TEMPLATES_DIR / "README.md").read_text() + for cheatsheet_name in ( + "elasticsearch-tunable-params.md", + "opensearch-tunable-params.md", + "solr-tunable-params.md", + ): + assert cheatsheet_name in text, ( + f"samples/templates/README.md is missing a link to {cheatsheet_name} " + "— operators need the cross-reference." + ) diff --git a/docs/00_overview/DASHBOARD.md b/docs/00_overview/DASHBOARD.md index b4fd509f..21dc8ab2 100644 --- a/docs/00_overview/DASHBOARD.md +++ b/docs/00_overview/DASHBOARD.md @@ -7,7 +7,7 @@ _Top-level index across MVP1 → GA v1+ as of **2026-06-02**. Click a release na | Release | Theme | Progress | Status | |---|---|---|---| | [MVP1 / v0.1](MVP1_DASHBOARD.md) | The Loop | 94 / 94 scoped done | **Complete** | -| [MVP2 / v0.2](MVP2_DASHBOARD.md) | Three-Engine + Real Signals | 10 / 22 scoped done · 24 remaining | **In progress** | +| [MVP2 / v0.2](MVP2_DASHBOARD.md) | Three-Engine + Real Signals | 10 / 22 scoped done · 25 remaining | **In progress** | | MVP3 / v0.3 | Observable | — | **Not yet scoped** | | GA v1 / v1.0 | Production-ready | — | **Not yet scoped** | diff --git a/docs/00_overview/MVP2_DASHBOARD.md b/docs/00_overview/MVP2_DASHBOARD.md index 09179033..b4bc5c1d 100644 --- a/docs/00_overview/MVP2_DASHBOARD.md +++ b/docs/00_overview/MVP2_DASHBOARD.md @@ -20,15 +20,15 @@ Plan approved; run /impl-execute to ship | Metric | Value | |---|---| -| Filed under MVP2 | **41** folders total (done + specced not-done + idea backlog + bugs) | +| Filed under MVP2 | **42** folders total (done + specced not-done + idea backlog + bugs) | | Specced features done | **10 / 22** (45%) — of features *past the idea stage* (those with a spec); the idea backlog below is NOT in this denominator, so 100% ≠ release complete | -| Pending work | **29** items (every not-done feat/infra/chore/bug across all priorities) | +| Pending work | **30** items (every not-done feat/infra/chore/bug across all priorities) | | → P0 — do next | **0** unblocking / paying daily cost | | → P1 | **1** high-value, ready when P0 clears | -| → P2 (default) | 24 important to file, not blocking | +| → P2 (default) | 25 important to file, not blocking | | → Backlog | 4 captured for record, not planned | -| Open bugs | 8 | -| Legacy "Path to MVP2" | 24 items — scoped-not-done + bugs + chore-ideas only (excludes feat/infra ideas) | +| Open bugs | 9 | +| Legacy "Path to MVP2" | 25 items — scoped-not-done + bugs + chore-ideas only (excludes feat/infra ideas) | | Backlog ideas | 5 idea-only feat/infra (not yet scoped into MVP2) | | In flight | 0 feature(s) actively shipping | @@ -78,7 +78,7 @@ _None._ _None._ -### Idea (15) +### Idea (16) | # | Priority | Feature | Type | One-liner | Depends on | Status | |---|---|---|---|---|---|---| @@ -92,11 +92,12 @@ _None._ | 8 | P2 | [bug_relyloop_spec_ubi_section_drift](planned_features/02_mvp2/bug_relyloop_spec_ubi_section_drift/idea.md) | Bug | [`docs/00_overview/relyloop-spec.md`](relyloop-spec.md) §"Click-derived judgments — OpenSearch UBI as the engine-neutral primary path" (line ~706) carries two staleness bugs from the 2026-05-27 releas | — | Idea — captured during `feat_ubi_judgments` preflight (2026-05-29) | | 9 | P2 | [bug_reseed_failure_blocks_retry_arq_singleton_dedup](planned_features/02_mvp2/bug_reseed_failure_blocks_retry_arq_singleton_dedup/idea.md) | Bug | `run_demo_reseed` is enqueued with a fixed Arq job id `demo_reseed:singleton` (the singleton concurrency guard). When a run reaches a terminal state, Arq stores its **result** under `arq:result:demo_r | — | Idea — tangential discovery while verifying `fix(demo): add Solr (8983) to the reseed engine host-URL mapping` (branch `feat_demo_reseed_solr_and_steplog`) | | 10 | P2 | [bug_seed_meaningful_demos_silent_bulk_errors](planned_features/02_mvp2/bug_seed_meaningful_demos_silent_bulk_errors/idea.md) | Bug | [`scripts/seed_meaningful_demos.py:917-935`](../../scripts/seed_meaningful_demos.py#L917-L935) bulk-indexes 1000 Amazon ESCI products into a dedicated index per demo scenario: | — | Idea — captured during `bug_smoke_seed_es_unavailable_shards_race` Phase 2.5 tangential sweep | -| 11 | P2 | [bug_webhook_concurrent_merge_race_timing_sensitive](planned_features/02_mvp2/bug_webhook_concurrent_merge_race_timing_sensitive/idea.md) | Bug | Idea — surfaced during `bug_demo_clusters_unreachable_in_healthz` PR #236 CI. | — | Idea — surfaced during `bug_demo_clusters_unreachable_in_healthz` PR #236 CI. | -| 12 | Backlog | [feat_fts_rank_ordering](planned_features/02_mvp2/feat_fts_rank_ordering/idea.md) | Feature | `feat_data_table_primitive` shipped filter-only FTS — `?q=foo` matches rows where `search_vector @@ plainto_tsquery('english', 'foo')` is true but orders results by `created_at DESC, id DESC` (the def | — | Idea — deferred from `feat_data_table_primitive` (MVP1) per spec §16. | -| 13 | Backlog | [infra_arq_subprocess_test](planned_features/02_mvp2/infra_arq_subprocess_test/idea.md) | Infra | Idea (deferred from `feat_study_lifecycle` Phase 2 / PR #25 final GPT-5.5 review). Still applicable as of 2026-05-14: the three in-process tests cited below still cover the resume contract correctly; | — | Idea (deferred from `feat_study_lifecycle` Phase 2 / PR #25 final GPT-5.5 review). Still applicable as of 2026-05-14: the three in-process tests cited below still cover the resume contract correctly; a subprocess test would add a narrow Arq-version-regression guard. | -| 14 | Backlog | [chore_auto_followup_parent_advisory_lock](planned_features/02_mvp2/chore_auto_followup_parent_advisory_lock/idea.md) | Chore | The shipped `feat_auto_followup_studies` worker uses a two-layer idempotency scheme: | — | Idea — captured as a standalone file to resolve broken cross-references in `feat_auto_followup_studies` D-11 + plan F2 + `bug_auto_followup_completed_parent_stop_chain_race/idea.md`. The slug was coined 2026-05-24 in D-11 but only existed as descriptive prose across other documents until now. | -| 15 | Backlog | [bug_chat_long_conversation_truncation](planned_features/02_mvp2/bug_chat_long_conversation_truncation/idea.md) | Bug | [`backend/app/services/agent_chat.send_user_message`](../../backend/app/services/agent_chat.py) defensively caps the OpenAI history at the most recent `HISTORY_MAX_MESSAGES = 100` messages… | — | Held for MVP2 (decided 2026-05-13). Folder renamed with `_mvp2` suffix to make the deferral visible at-a-glance in `ls docs/00_overview/planned_features/`. Resume work when MVP2 starts — no technical dependency on MVP2 infra (audit_log is N/A; Langfuse is convenience only); the deferral is scope discipline + zero current impact (latent bug, no operator has hit the 100-message cap). | +| 11 | P2 | [bug_studies_detail_vitest_intermittent_timeout](planned_features/02_mvp2/bug_studies_detail_vitest_intermittent_timeout/idea.md) | Bug | Under the full `pnpm test` run (`vitest run`, default worker pool), the Study-detail-page render test sometimes blocks past the 5 s `testTimeout` default — but the test itself is data-driven from mock | — | Idea — captured during `chore_template_library_expansion` post-impl tangential sweep | +| 12 | P2 | [bug_webhook_concurrent_merge_race_timing_sensitive](planned_features/02_mvp2/bug_webhook_concurrent_merge_race_timing_sensitive/idea.md) | Bug | Idea — surfaced during `bug_demo_clusters_unreachable_in_healthz` PR #236 CI. | — | Idea — surfaced during `bug_demo_clusters_unreachable_in_healthz` PR #236 CI. | +| 13 | Backlog | [feat_fts_rank_ordering](planned_features/02_mvp2/feat_fts_rank_ordering/idea.md) | Feature | `feat_data_table_primitive` shipped filter-only FTS — `?q=foo` matches rows where `search_vector @@ plainto_tsquery('english', 'foo')` is true but orders results by `created_at DESC, id DESC` (the def | — | Idea — deferred from `feat_data_table_primitive` (MVP1) per spec §16. | +| 14 | Backlog | [infra_arq_subprocess_test](planned_features/02_mvp2/infra_arq_subprocess_test/idea.md) | Infra | Idea (deferred from `feat_study_lifecycle` Phase 2 / PR #25 final GPT-5.5 review). Still applicable as of 2026-05-14: the three in-process tests cited below still cover the resume contract correctly; | — | Idea (deferred from `feat_study_lifecycle` Phase 2 / PR #25 final GPT-5.5 review). Still applicable as of 2026-05-14: the three in-process tests cited below still cover the resume contract correctly; a subprocess test would add a narrow Arq-version-regression guard. | +| 15 | Backlog | [chore_auto_followup_parent_advisory_lock](planned_features/02_mvp2/chore_auto_followup_parent_advisory_lock/idea.md) | Chore | The shipped `feat_auto_followup_studies` worker uses a two-layer idempotency scheme: | — | Idea — captured as a standalone file to resolve broken cross-references in `feat_auto_followup_studies` D-11 + plan F2 + `bug_auto_followup_completed_parent_stop_chain_race/idea.md`. The slug was coined 2026-05-24 in D-11 but only existed as descriptive prose across other documents until now. | +| 16 | Backlog | [bug_chat_long_conversation_truncation](planned_features/02_mvp2/bug_chat_long_conversation_truncation/idea.md) | Bug | [`backend/app/services/agent_chat.send_user_message`](../../backend/app/services/agent_chat.py) defensively caps the OpenAI history at the most recent `HISTORY_MAX_MESSAGES = 100` messages… | — | Held for MVP2 (decided 2026-05-13). Folder renamed with `_mvp2` suffix to make the deferral visible at-a-glance in `ls docs/00_overview/planned_features/`. Resume work when MVP2 starts — no technical dependency on MVP2 infra (audit_log is N/A; Langfuse is convenience only); the deferral is scope discipline + zero current impact (latent bug, no operator has hit the 100-message cap). | ## Dependency graph diff --git a/docs/00_overview/dashboard.html b/docs/00_overview/dashboard.html index de601348..68ff681e 100644 --- a/docs/00_overview/dashboard.html +++ b/docs/00_overview/dashboard.html @@ -392,7 +392,7 @@

Releases

Three-Engine + Real Signals
-
10 / 22 scoped done · 24 remaining
+
10 / 22 scoped done · 25 remaining
In progress
diff --git a/docs/00_overview/mvp2_dashboard.html b/docs/00_overview/mvp2_dashboard.html index 11dd1047..d997fbcf 100644 --- a/docs/00_overview/mvp2_dashboard.html +++ b/docs/00_overview/mvp2_dashboard.html @@ -398,17 +398,17 @@

MVP2 Progress

Specced features done
10 / 22
-
45% specced · 41 filed under MVP2
+
45% specced · 42 filed under MVP2
Pending work
-
29
+
30
every not-done feat/infra/chore/bug across all priorities
Open bugs
-
8
+
9
tracked bug_* idea files
@@ -425,7 +425,7 @@

MVP2 Progress

P2 (default)
-
24
+
25
important to file, not blocking
@@ -435,7 +435,7 @@

MVP2 Progress

Legacy "Path to MVP2"
-
24
+
25
scoped not-done + bugs + chore-ideas only (excludes feat/infra ideas)
@@ -463,7 +463,7 @@

Pipeline

-

Idea 15

+

Idea 16

@@ -595,6 +595,19 @@

Idea 15

+
+ +
+ Bug + P2 + +
+
Under the full `pnpm test` run (`vitest run`, default worker pool), the Study-detail-page render test sometimes blocks past the 5 s `testTimeout` default — but the test itself is data-driven from mock
+ + +
+ +
diff --git a/docs/00_overview/planned_features/02_mvp2/bug_studies_detail_vitest_intermittent_timeout/idea.md b/docs/00_overview/planned_features/02_mvp2/bug_studies_detail_vitest_intermittent_timeout/idea.md new file mode 100644 index 00000000..e6375e84 --- /dev/null +++ b/docs/00_overview/planned_features/02_mvp2/bug_studies_detail_vitest_intermittent_timeout/idea.md @@ -0,0 +1,43 @@ +# Idea — `studies/[id]/page.test.tsx` intermittently times out under full-suite vitest run + +**Date:** 2026-06-02 +**Status:** Idea — captured during `chore_template_library_expansion` post-impl tangential sweep +**Origin:** Noticed while running `pnpm test` as the Story 3.1 (FR-7) pre-push gate during PR `chore/template-library-expansion`. First full-suite run: `src/__tests__/app/studies/[id]/page.test.tsx > Study detail page > renders header, trials table, and digest panel for a completed study` failed with `Test timed out in 5000ms` after 5042ms wall-clock + a JSDOM `Not implemented: navigation to another Document` log line. Second full-suite run (no code change): 1003 / 1003 passed. The same test, run in isolation, passes in <1 s (`pnpm test -- run src/__tests__/app/studies/[id]/page.test.tsx`). + +**Priority:** P2 — intermittent. CI is unlikely to flag it consistently (vitest's worker pool dispatches files in different orders run-to-run), but the noise will burn future operator time on false-alarm investigations until the underlying race is fixed. + +## Problem + +Under the full `pnpm test` run (`vitest run`, default worker pool), the Study-detail-page render test sometimes blocks past the 5 s `testTimeout` default — but the test itself is data-driven from mocked fixtures and shouldn't be doing real I/O. The JSDOM `Not implemented: navigation to another Document` log line strongly suggests something in the test environment (or a sibling test that ran in the same worker before it) is triggering a `window.location` assignment / form submission that JSDOM can't honour. + +The failing assertion site: + +- File: [`ui/src/__tests__/app/studies/[id]/page.test.tsx:88`](ui/src/__tests__/app/studies/[id]/page.test.tsx) +- Test: `'renders header, trials table, and digest panel for a completed study'` +- Default vitest `testTimeout`: 5000 ms (see `ui/vitest.config.ts` if a project-level override exists). + +## Why it wasn't fixed inline on PR `chore/template-library-expansion` + +Per CLAUDE.md's tangential-discoveries rubric: + +- **Fix path uncertain.** Resolving an intermittent test-isolation flake usually means identifying which OTHER test (running in the same worker before this one) is leaving a stray timer / pending fetch / window listener that this test then trips over. That investigation is open-ended and not a 60-min path — the offending sibling could be anywhere in 135 test files. +- **Cross-subsystem.** The PR's scope is content + docs + tests for the template library. Investigating a Study-detail-page test (an entirely separate UI surface, not touched by the PR) would conflate scope. +- **Pre-existing.** The flake reproduces on `main` — verified by running the full UI suite twice; the second run was green. My PR's only UI change is an additive optional `learnMoreHref` prop on `InfoTooltip` + a new `template-descriptions.ts` map + a Step-3 modal summary block. None of those touch the Study detail page's rendering paths. + +The PR proceeded on the green second-run. + +## Investigation paths (for whoever picks this up) + +1. **Pin the offender via worker-isolation.** Run `pnpm test --pool=forks --poolOptions.forks.singleFork=true` — if the test passes deterministically, the cause is cross-test state in the default worker pool. `git bisect`-style binary-search over the test-file list will identify the polluter. +2. **Hunt for in-test navigation.** Grep the suite for `window.location`, `form.submit()`, anchor clicks with `target="_self"`, and `Link` navigations not wrapped in a mock router. The JSDOM `Not implemented: navigation to another Document` log is emitted from JSDOM's URL-changing code paths. +3. **Raise `testTimeout` defensively.** Last-resort patch if the root cause stays elusive: bump the suite's `testTimeout` to 10000 ms (or set it per-file in the affected test). This is the "deferral" fix — it covers the symptom but not the cause. + +## Acceptance signal + +- `pnpm test` (default worker pool, all 136 files) runs 10 consecutive times with zero timeouts on the `studies/[id]/page.test.tsx` cases. +- OR: the polluting sibling test is fixed and a one-line regression assertion is added to prevent recurrence. + +## Cross-links + +- Tangential to: `chore_template_library_expansion` PR `chore/template-library-expansion` (2026-06-02). +- Reminder of the CLAUDE.md "test-isolation bug" failure-mode entry under `## Tangential discoveries`: re-running a flake without investigating is the trap; this idea file IS the investigation paper trail. diff --git a/docs/00_overview/planned_features/02_mvp2/chore_template_library_expansion/implementation_plan.md b/docs/00_overview/planned_features/02_mvp2/chore_template_library_expansion/implementation_plan.md index 352e8ba1..7f771ae3 100644 --- a/docs/00_overview/planned_features/02_mvp2/chore_template_library_expansion/implementation_plan.md +++ b/docs/00_overview/planned_features/02_mvp2/chore_template_library_expansion/implementation_plan.md @@ -1,7 +1,7 @@ # Implementation Plan — Curated query-template library + per-engine tunable-params cheatsheets **Date:** 2026-06-02 -**Status:** Ready for Execution +**Status:** In Progress (PR open; 8 of 8 stories complete, gates green; finalize on merge) **Primary spec:** [`feature_spec.md`](feature_spec.md) **Policy source(s):** [`docs/01_architecture/adapters.md`](../../../../01_architecture/adapters.md) §"Cross-engine parameter naming"; [`samples/templates/README.md`](../../../../../samples/templates/README.md); [`docs/06_vendor_docs/README.md`](../../../../06_vendor_docs/README.md) @@ -405,14 +405,14 @@ from backend.app.domain.study.search_space import SearchSpace # parse .search_s ## 9) Execution tracker ### Current sprint -- [ ] Story 1.1 — ES/OS templates + starter spaces + README registration blocks -- [ ] Story 1.2 — Solr templates + starter spaces + solr/README.md -- [ ] Story 1.3 — render-validation + doc-consistency tests -- [ ] Story 2.1 — ES cheatsheet (+ kNN/hybrid ref snippets) -- [ ] Story 2.2 — OpenSearch cheatsheet (+ normalization-processor hybrid snippet) -- [ ] Story 2.3 — Solr cheatsheet -- [ ] Story 2.4 — vendor README index + samples README + tutorial -- [ ] Story 3.1 — FR-7 (conditional: ship or cut with decision-log note) +- [x] Story 1.1 — ES/OS templates + starter spaces + README registration blocks +- [x] Story 1.2 — Solr templates + starter spaces + solr/README.md +- [x] Story 1.3 — render-validation + doc-consistency tests +- [x] Story 2.1 — ES cheatsheet (+ kNN/hybrid ref snippets) +- [x] Story 2.2 — OpenSearch cheatsheet (+ normalization-processor hybrid snippet) +- [x] Story 2.3 — Solr cheatsheet +- [x] Story 2.4 — vendor README index + samples README + tutorial +- [x] Story 3.1 — FR-7 **SHIPPED** (feasibility gate passed: client-side `template-descriptions.ts` + `cheatsheetUrlFor` resolver + optional `learnMoreHref` prop on `InfoTooltip` — no migration, no new endpoint, no shared-tooltip refactor) ### Blocked items - None. diff --git a/docs/00_overview/planned_features/02_mvp2/chore_template_library_expansion/pipeline_status.md b/docs/00_overview/planned_features/02_mvp2/chore_template_library_expansion/pipeline_status.md index fffa24db..bf58a68b 100644 --- a/docs/00_overview/planned_features/02_mvp2/chore_template_library_expansion/pipeline_status.md +++ b/docs/00_overview/planned_features/02_mvp2/chore_template_library_expansion/pipeline_status.md @@ -1,5 +1,7 @@ # Pipeline Status — Curated query-template library + per-engine tunable-params cheatsheets +**Release:** mvp2 + ## Idea - Status: Complete - File: idea.md @@ -20,4 +22,16 @@ - Phases covered: 1 of 1 (single-phase) ## Implementation -- Status: Not started +- Status: PR created (chore/template-library-expansion branch) +- Branch: `chore/template-library-expansion` +- Date: 2026-06-02 +- Stories completed: 8 of 8 (Epic 1: 1.1 / 1.2 / 1.3 — runnable templates + render + invariant tests; Epic 2: 2.1 / 2.2 / 2.3 / 2.4 — three cheatsheets + index + tutorial + cheatsheet doc-consistency test; Epic 3: 3.1 — FR-7 **SHIPPED** client-side: `template-descriptions.ts` + optional `learnMoreHref` prop on `InfoTooltip` + modal wiring) +- Gates green locally: + - `make lint && make typecheck` clean + - `make test-unit` — 2145 passed (incl. 30 new template-library tests + 14 new cheatsheet tests + 2 new Solr library render tests + 9 new ES library render tests) + - `make test-contract` — 327 passed (66 unrelated Postgres-unreachable skips) + - `pnpm test` — 1003 passed (incl. 5 new template-descriptions vitest cases) + - `pnpm lint && pnpm typecheck && pnpm build` clean + - `ruff format --check backend/` clean (CI parity) +- No migration (Alembic head stays at `0022_solr_engine_auth_check`). +- AC-3 byte-stability: the four existing demo templates untouched (asserted by `test_demo_template_unchanged`). diff --git a/docs/06_vendor_docs/README.md b/docs/06_vendor_docs/README.md index 71fb3ada..7fd6455f 100644 --- a/docs/06_vendor_docs/README.md +++ b/docs/06_vendor_docs/README.md @@ -6,12 +6,22 @@ Engine-, provider-, or vendor-specific references, adapter notes, version quirks | Doc | What it covers | Used by | |---|---|---| +| [`elasticsearch-tunable-params.md`](elasticsearch-tunable-params.md) | Per-knob Elasticsearch tuning reference — native + unified names, ranges, "when to tune", caveats, template back-links; plus kNN + native `rrf`-retriever hybrid reference snippets (8.11+) | `chore_template_library_expansion`; operators tuning the runnable [template library](../../samples/templates/) on Elasticsearch | +| [`opensearch-tunable-params.md`](opensearch-tunable-params.md) | Per-knob OpenSearch reference — most of the lexical surface mirrors ES (cross-references rather than duplicates); the hybrid section documents OpenSearch's normalization-processor construct (NOT the ES `rrf` retriever) | `chore_template_library_expansion`; operators tuning on OpenSearch 2.x / 3.x | +| [`solr-tunable-params.md`](solr-tunable-params.md) | Per-knob Apache Solr reference grounded in the checked-in [`solr-9/`](solr-9/) + [`solr-10/`](solr-10/) ref-guide source; unified-name → native pivots (`field_boosts`→`qf`, `boost_fn`→`bf`/`boost`, …); kNN + hybrid documented as out-of-scope for the library | `chore_template_library_expansion`; operators tuning on Solr 9.x / 10.x | | [`github-branch-protection.md`](github-branch-protection.md) | Two procedures (modern Rulesets + classic Branch Protection) for requiring CI status checks before merge to `main`; the three exact check names for the `relyloop` repo; verification + gotchas | `infra_foundation` plan §7.5 manual handoff #3; every operator who needs to update branch rules | | [`github-pages-custom-domain.md`](github-pages-custom-domain.md) | Publishing the `website/` MkDocs site to GitHub Pages at the apex domain **relyloop.com** via GoDaddy DNS: the four apex `A` IPs (+ optional IPv6 `AAAA`), `www` CNAME, the `website/docs/CNAME` mechanism, Let's Encrypt cert timing + the "Enforce HTTPS unavailable" remove-and-re-add remedy, GoDaddy parking-page gotcha, DoH/curl verification, and the verified relyloop.com zone | `deploy-docs.yml`; any operator wiring or debugging the relyloop.com custom domain | -| [`solr-10/`](solr-10/) | Apache Solr 10.0 ref-guide pages (asciidoc source, tag `releases/solr/10.0.0`) — matches the `solr:10.0` image. Module loading (`SOLR_MODULES`, no ``), configsets + UPLOAD API, LTR, auth | `infra_adapter_solr` (MVP2 Solr adapter + Compose infra) | -| [`solr-9/`](solr-9/) | Apache Solr 9.9 ref-guide pages (asciidoc source, tag `releases/solr/9.9.0`) for cross-version comparison | `infra_adapter_solr` — confirm which behaviours differ between 9.x and 10 | +| [`solr-10/`](solr-10/) | Apache Solr 10.0 ref-guide pages (asciidoc source, tag `releases/solr/10.0.0`) — matches the `solr:10.0` image. Module loading (`SOLR_MODULES`, no ``), configsets + UPLOAD API, LTR, auth | `infra_adapter_solr` (MVP2 Solr adapter + Compose infra); cited by [`solr-tunable-params.md`](solr-tunable-params.md) | +| [`solr-9/`](solr-9/) | Apache Solr 9.9 ref-guide pages (asciidoc source, tag `releases/solr/9.9.0`) for cross-version comparison | `infra_adapter_solr` — confirm which behaviours differ between 9.x and 10; cited by [`solr-tunable-params.md`](solr-tunable-params.md) | | [`relevance-tools/`](relevance-tools/) | Distilled capability snapshots (with upstream URLs + access dates) of the adjacent/competing relevance tools — OpenSearch SRW + Relevance Agent, Quepid, RRE, Chorus, Elasticsearch native, Splainer. Competitive-landscape references, not integration targets | Evidence base for [`docs/07_research/comparison.md`](../07_research/comparison.md); refresh when a tool ships a release that flips a comparison cell | +The `*-tunable-params.md` cheatsheets are a **separate kind of doc** from +the `elasticsearch-9x.md` / `opensearch-2x.md` version-quirk files +reserved below — the cheatsheets enumerate every tunable knob's native ++ unified name, range, "when to tune", and caveats grounded in upstream +docs; the version-quirk files (when they land) document workarounds the +adapter must apply. Don't merge the two. + ### Solr docs detail Both dirs hold the same 9 ref-guide pages as **asciidoc source** from diff --git a/docs/06_vendor_docs/elasticsearch-tunable-params.md b/docs/06_vendor_docs/elasticsearch-tunable-params.md new file mode 100644 index 00000000..eb9ec035 --- /dev/null +++ b/docs/06_vendor_docs/elasticsearch-tunable-params.md @@ -0,0 +1,202 @@ + + +# Elasticsearch tunable parameters + +Per-knob reference for the Elasticsearch tunable parameters exposed by +RelyLoop's runnable [template library](../../samples/templates/). Every +section covers: the native ES name, RelyLoop's unified name (when they +differ), valid range / choices, when to tune, common caveats, and the +templates in the library that declare the knob. + +**Engine versions covered:** Elasticsearch **8.11+** and **9.x**. Older +ES versions are out of scope (see `docs/01_architecture/adapters.md` +§"Engine version support"). For the OpenSearch-specific divergences, +see [`opensearch-tunable-params.md`](opensearch-tunable-params.md). + +**Companion docs:** + +- [`adapters.md`](../01_architecture/adapters.md) §"Cross-engine parameter naming" — the canonical unified-name → native-name table this cheatsheet expands. +- [`samples/templates/README.md`](../../samples/templates/README.md) — the runnable template library + per-template registration blocks. + +--- + +## Unified vocabulary (8 cross-engine params) + +The cross-engine vocabulary in +[`adapters.md`](../01_architecture/adapters.md) §"Cross-engine parameter +naming" defines 8 unified names that work the same way on every adapter. +Each section below pairs the unified name with the ES native key. + +### `field_boosts` (per-field weights) + +- **Native ES name:** the `fields` array on a `multi_match` query, e.g. `"fields": ["title^2.0", "description^1.0"]`. +- **Range:** float ≥ 0 per field. Effective signal range: 0.1–10.0; beyond that the boost dominates the tf-idf signal. +- **When to tune:** when one of your fields holds more discriminative tokens than the others (typically `title` > `description` > `bullet_points`). +- **Caveats:** the boost is multiplicative on the per-field score, NOT a global weight — extreme values (e.g. 100×) effectively short-circuit ranking to a single field. +- **Templates that use this param:** `multi_match_basic.j2` (via `title_boost`, `description_boost`, `bullet_points_boost`), `function_score_decay.j2` (same), `bool_boosted.j2` (same), `rescore_phrase.j2` (same). +- **Source:** [ES `multi_match` query reference](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/query-dsl-multi-match-query.html) (accessed 2026-06-02). + +### `phrase_field_boosts` + +- **Native ES name:** there is no single ES param — phrase emphasis is built either via a `match_phrase` clause inside a `bool` query OR via a `rescore` block over a `match_phrase` (see `rescore_phrase.j2`). +- **Range:** N/A — it's a clause shape, not a numeric knob. +- **When to tune:** when exact phrase matches are an important relevance signal (long product titles, named entities). +- **Caveats:** ES doesn't expose a single `phrase_field_boosts` parameter — the unified concept maps to a `rescore` over a `match_phrase` clause. See `rescore_phrase.j2` for the canonical shape. +- **Templates that use this param:** `rescore_phrase.j2` (implicitly via the phrase-rescore clause). + +### `tie_breaker` + +- **Native ES name:** `tie_breaker`. +- **Range:** float in `[0.0, 1.0]`. `0.0` = pure best-match-only; `1.0` = sum of all field scores. +- **When to tune:** when documents matching multiple fields should rank above documents that only match one — typical for product search with semi-redundant fields. +- **Caveats:** only meaningful with `type: best_fields` or `most_fields` multi_match; ignored on `cross_fields`. +- **Templates that use this param:** `multi_match_basic.j2` (via `tie_breaker`). +- **Source:** [ES `multi_match` `tie_breaker`](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/query-dsl-multi-match-query.html#type-best-fields) (accessed 2026-06-02). + +### `min_should_match` + +- **Native ES name:** `minimum_should_match` on a `bool` query. +- **Range:** integer (`1`, `2`, …), percentage (`50%`, `75%`), or arithmetic syntax (`2<-25% 9<-3`). +- **When to tune:** to balance recall vs. precision on multi-clause `bool` queries. +- **Caveats:** the arithmetic syntax only kicks in once the should-clause count exceeds the threshold — on short queries ES silently falls back to a `100%`-equivalent. +- **Templates that use this param:** `bool_boosted.j2` (via `min_should_match`). +- **Source:** [ES `minimum_should_match` reference](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/query-dsl-minimum-should-match.html) (accessed 2026-06-02). + +### `fuzziness` + +- **Native ES name:** `fuzziness`. +- **Range:** `"0"`, `"1"`, `"2"`, or `"AUTO"`. (Numeric integers are also accepted on `match` clauses.) +- **When to tune:** on noisy user input (typos, OCR text, free-text queries). `AUTO` adapts edit distance to term length. +- **Caveats:** `AUTO` is markedly slower on long queries than a fixed edit distance because every term gets a per-length expansion. +- **Templates that use this param:** `multi_match_basic.j2` (via `fuzziness`). +- **Source:** [ES `fuzziness` reference](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/common-options.html#fuzziness) (accessed 2026-06-02). + +### `slop` + +- **Native ES name:** `slop` on `match_phrase` / `match_phrase_prefix` queries. +- **Range:** integer ≥ 0. Practical range: 0–5. +- **When to tune:** when an exact phrase is too strict but token-order should still matter (e.g. "blue sofa" → "sofa, blue"). +- **Caveats:** `slop` is a **no-op** on `multi_match best_fields` — that's why `multi_match_basic.j2` deliberately omits it. The phrase-slop knob lives on `rescore_phrase.j2` where the rescore-pass query IS a `match_phrase`. +- **Templates that use this param:** `rescore_phrase.j2` (via `rescore_phrase_slop`). +- **Source:** [ES `match_phrase` `slop`](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/query-dsl-match-query-phrase.html) (accessed 2026-06-02). + +### `boost_fn` (boost function) + +- **Native ES name:** `function_score` query with a `functions` array (multiplicative by default; additive when the combine semantics are explicit). +- **Range:** the function family is open-ended (`gauss`, `linear`, `exp` decays; `field_value_factor`; arbitrary `script_score`). +- **When to tune:** any time a non-lexical signal (recency, popularity, price) should influence rank order. +- **Caveats:** combining `function_score` with `rescore` produces hard-to-reason-about score multipliers; pick one boosting layer. +- **Templates that use this param:** `function_score_decay.j2` (a `gauss` decay over `created_at`). +- **Source:** [ES `function_score`](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/query-dsl-function-score-query.html) (accessed 2026-06-02). + +### `rerank_model` + +- **Native ES name:** `rescore` with an LTR model OR (8.13+) the native `learning_to_rank` retriever. +- **Range:** depends on the LTR model deployed. +- **When to tune:** when an offline-trained ranking model exists. +- **Caveats:** LTR model deployment is out of scope for the RelyLoop tuning loop (RelyLoop tunes query-time params, not model weights). +- **Templates that use this param:** none in the runnable library (no LTR-bearing template ships in MVP2). + +--- + +## ES-specific knobs (template library coverage) + +### `decay_scale`, `decay_offset`, `decay_decay` + +- **Native ES name:** `scale`, `offset`, `decay` parameters under a `gauss` / `linear` / `exp` decay function inside `function_score`. +- **Range:** + - `scale` — a duration string (`"30d"`, `"180d"`) or numeric distance. Controls the half-life-style falloff distance. + - `offset` — same unit as `scale`. The plateau distance before decay starts. Default `0`. + - `decay` — float in `(0, 1)`. The value the function returns at `scale` distance from `origin`. Default `0.5`. +- **When to tune:** when freshness / proximity should influence rank. `scale` controls *how fast* the boost fades; `decay` controls *how steep* the curve is at `scale` distance. +- **Caveats:** `scale` must use the same unit family as the source field (date-typed → date-math; numeric → bare numeric). Crossing the boundary silently produces a no-op decay. +- **Templates that use this param:** `function_score_decay.j2`. +- **Source:** [ES decay functions](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/query-dsl-function-score-query.html#function-decay) (accessed 2026-06-02). + +### `rescore_window_size`, `rescore_query_weight`, `rescore_phrase_slop` + +- **Native ES name:** `window_size`, `query_weight` (and the implicit `rescore_query_weight`), and `slop` inside a `rescore` block. +- **Range:** + - `window_size` — integer ≥ 1. Practical range: 10–500. + - `query_weight` — float ≥ 0. The relative weight of the first-pass score in the combined output. + - `phrase_slop` (which maps onto the rescore-clause's `slop`) — integer ≥ 0. Practical range: 0–5. +- **When to tune:** when a fast first-pass query gets recall right but exact-phrase matches should be promoted within the top-N. +- **Caveats:** `window_size` is per-shard; setting it larger than `from + size` wastes computation. Combined `query_weight` + the implicit rescore weight controls how aggressively the rescore overrides the first pass. +- **Templates that use this param:** `rescore_phrase.j2`. +- **Source:** [ES query rescorer](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/filter-search-results.html#rescore) (accessed 2026-06-02). + +--- + +## Vector & hybrid (reference shapes) + +> The following templates are **reference snippets**, not runnable +> templates — they require a query-vector mechanism the RelyLoop trial +> runner does not currently inject (the render context is exactly +> `{**params, "query_text": query_text}`; no embedding pipeline runs). +> They graduate to runnable templates if a future feature wires +> query-vector injection. See `samples/templates/README.md` "What's NOT +> here (and why)" for the scoping decision. + +### kNN (Elasticsearch native) + +ES 8.11+ exposes `knn` as both a top-level search clause and as the +dense-vector retriever building block for the `rrf` retriever (below). + +```json +{ + "knn": { + "field": "embedding", + "query_vector": "", + "k": 50, + "num_candidates": 200, + "boost": 1.0 + } +} +``` + +Key knobs (cite [ES kNN search ref](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/knn-search.html), accessed 2026-06-02): + +| Knob | Range | When to tune | +|---|---|---| +| `k` | int ≥ 1 | how many neighbours to return | +| `num_candidates` | int ≥ `k` | how wide the per-shard scan is — recall/latency knob | +| `boost` | float | multiplier on the kNN score relative to other clauses | + +### Hybrid (Elasticsearch native `rrf` retriever — 8.11+) + +```json +{ + "retriever": { + "rrf": { + "retrievers": [ + {"standard": {"query": {"multi_match": {"query": "", "fields": ["title", "description"]}}}}, + {"knn": {"field": "embedding", "query_vector": "", "k": 50, "num_candidates": 200}} + ], + "rank_window_size": 100, + "rank_constant": 60 + } + } +} +``` + +**This is the Elasticsearch-only construct.** OpenSearch 2.x does NOT +ship the `rrf` retriever — its hybrid surface is a search-pipeline +normalization processor (see +[`opensearch-tunable-params.md`](opensearch-tunable-params.md) §"Hybrid"). +The two are **not interchangeable** — copying the snippet above into +OpenSearch produces a 400. + +**Source:** [ES `rrf` retriever ref](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/rrf.html) (accessed 2026-06-02). + +--- + +## See also + +- [`samples/templates/README.md`](../../samples/templates/README.md) — runnable ES/OS template library +- [`opensearch-tunable-params.md`](opensearch-tunable-params.md) — OpenSearch-specific divergences (hybrid construct in particular) +- [`solr-tunable-params.md`](solr-tunable-params.md) — Apache Solr cheatsheet +- [`docs/01_architecture/adapters.md`](../01_architecture/adapters.md) — `SearchAdapter` Protocol + cross-engine vocabulary diff --git a/docs/06_vendor_docs/opensearch-tunable-params.md b/docs/06_vendor_docs/opensearch-tunable-params.md new file mode 100644 index 00000000..28500b11 --- /dev/null +++ b/docs/06_vendor_docs/opensearch-tunable-params.md @@ -0,0 +1,207 @@ + + +# OpenSearch tunable parameters + +Per-knob reference for the OpenSearch tunable parameters exposed by +RelyLoop's runnable [template library](../../samples/templates/). Most +of the lexical surface mirrors Elasticsearch (`multi_match`, +`function_score`, `bool`, `rescore`) — the divergences live in **hybrid +search** (no `rrf` retriever) and in version-specific defaults. + +**Engine versions covered:** OpenSearch **2.x** (baseline equivalent to +ES 7.10) and **3.x**. For the lexical knobs that ARE shared with ES, +this doc cross-references +[`elasticsearch-tunable-params.md`](elasticsearch-tunable-params.md) +rather than duplicating the prose; the engine-divergence sections are +called out explicitly. + +**Companion docs:** + +- [`adapters.md`](../01_architecture/adapters.md) §"Cross-engine parameter naming" +- [`samples/templates/README.md`](../../samples/templates/README.md) +- [`elasticsearch-tunable-params.md`](elasticsearch-tunable-params.md) (~90% of the lexical surface is shared) + +--- + +## Unified vocabulary (8 cross-engine params) + +### `field_boosts` (per-field weights) + +- **Native OpenSearch name:** the `fields` array on `multi_match` — same shape as ES (`["title^2.0", "description^1.0"]`). +- **Range:** float ≥ 0 per field; practical range 0.1–10. +- **When to tune:** identical to ES — when one field carries more discriminative signal than the others. +- **Caveats:** none beyond the ES advice. Lexical DSL is byte-identical between OpenSearch 2.x and ES 8.11+ for `multi_match`. +- **Templates that use this param:** `multi_match_basic.j2`, `function_score_decay.j2`, `bool_boosted.j2`, `rescore_phrase.j2` (via `title_boost`, `description_boost`, `bullet_points_boost`). +- **Source:** [OpenSearch `multi_match`](https://opensearch.org/docs/2.13/query-dsl/full-text/multi-match/) (accessed 2026-06-02). + +### `phrase_field_boosts` + +- **Native OpenSearch name:** no single key — same as ES, the canonical implementation is a `rescore` clause over `match_phrase`. +- **Templates that use this param:** `rescore_phrase.j2`. + +### `tie_breaker` + +- **Native OpenSearch name:** `tie_breaker` (same key as ES). +- **Range:** float in `[0.0, 1.0]`. +- **When to tune:** same as ES — multi-field discrimination with semi-redundant fields. +- **Caveats:** only meaningful with `best_fields` / `most_fields` multi_match; ignored on `cross_fields`. No OpenSearch-specific divergence. +- **Templates that use this param:** `multi_match_basic.j2` (via `tie_breaker`). +- **Source:** [OpenSearch `multi_match` `tie_breaker`](https://opensearch.org/docs/2.13/query-dsl/full-text/multi-match/#best-fields) (accessed 2026-06-02). + +### `min_should_match` + +- **Native OpenSearch name:** `minimum_should_match` on a `bool` query. +- **Range:** integer, percentage, or arithmetic syntax — identical to ES. +- **Caveats:** identical to ES — arithmetic syntax requires ≥ 3 should-clauses to activate. +- **Templates that use this param:** `bool_boosted.j2` (via `min_should_match`). +- **Source:** [OpenSearch `minimum_should_match`](https://opensearch.org/docs/2.13/query-dsl/minimum-should-match/) (accessed 2026-06-02). + +### `fuzziness` + +- **Native OpenSearch name:** `fuzziness`. +- **Range:** `"0"`, `"1"`, `"2"`, `"AUTO"`. Same accepted shapes as ES. +- **Caveats:** identical to ES — `AUTO` adapts edit distance to term length and is slower on long queries. +- **Templates that use this param:** `multi_match_basic.j2` (via `fuzziness`). +- **Source:** [OpenSearch common options](https://opensearch.org/docs/2.13/query-dsl/full-text/index/#common-options) (accessed 2026-06-02). + +### `slop` + +- **Native OpenSearch name:** `slop` on `match_phrase` / `match_phrase_prefix`. +- **Range:** integer ≥ 0; practical 0–5. +- **Caveats:** identical to ES — `slop` is a no-op on `best_fields` multi_match. The runnable `rescore_phrase.j2` is the only template that declares it. +- **Templates that use this param:** `rescore_phrase.j2` (via `rescore_phrase_slop`). +- **Source:** [OpenSearch `match_phrase`](https://opensearch.org/docs/2.13/query-dsl/full-text/match-phrase/) (accessed 2026-06-02). + +### `boost_fn` (boost function) + +- **Native OpenSearch name:** `function_score` query with a `functions` array — same shape as ES. +- **Range:** decay families (`gauss`, `linear`, `exp`), `field_value_factor`, `script_score`. +- **Caveats:** OpenSearch's `script_score` uses **Painless** (same as ES); `function_score` script changes between OpenSearch 2.x and 3.x are minimal but check the version page. +- **Templates that use this param:** `function_score_decay.j2` (a `gauss` decay). +- **Source:** [OpenSearch `function_score`](https://opensearch.org/docs/2.13/query-dsl/compound/function-score/) (accessed 2026-06-02). + +### `rerank_model` + +- **Native OpenSearch name:** `learning_to_rank` query (OpenSearch LTR plugin) OR a `rescore` clause. +- **Caveats:** LTR model deployment is out of scope for the RelyLoop tuning loop (RelyLoop tunes query-time params). +- **Templates that use this param:** none in the runnable library. + +--- + +## OpenSearch-specific knobs (template library coverage) + +### `decay_scale`, `decay_offset`, `decay_decay` + +- **Native OpenSearch name:** `scale`, `offset`, `decay` under a `gauss` / `linear` / `exp` decay inside `function_score`. Identical shape to ES. +- **Range:** see `elasticsearch-tunable-params.md` §"`decay_scale`, `decay_offset`, `decay_decay`" — same defaults and same accepted unit families. +- **Caveats:** OpenSearch's date-math parser is the same lib as ES; cross-version differences appear only on `date_nanos` precision (rare in catalog search). +- **Templates that use this param:** `function_score_decay.j2`. +- **Source:** [OpenSearch decay functions](https://opensearch.org/docs/2.13/query-dsl/compound/function-score/#decay-functions) (accessed 2026-06-02). + +### `rescore_window_size`, `rescore_query_weight`, `rescore_phrase_slop` + +- **Native OpenSearch name:** `window_size`, `query_weight`, and rescore-clause `slop` — identical to ES. +- **Range:** same practical ranges (`window_size` 10–500; `query_weight` 0–5; `slop` 0–5). +- **Caveats:** identical to ES. Per-shard window math is the same. +- **Templates that use this param:** `rescore_phrase.j2`. +- **Source:** [OpenSearch rescore](https://opensearch.org/docs/2.13/search-plugins/rescore/) (accessed 2026-06-02). + +--- + +## Vector & hybrid (reference shapes) + +> The following templates are **reference snippets**, not runnable +> templates — they require a query-vector mechanism the RelyLoop trial +> runner does not currently inject (the render context is exactly +> `{**params, "query_text": query_text}`; no embedding pipeline runs). +> They graduate to runnable templates if a future feature wires +> query-vector injection. + +### kNN (OpenSearch native) + +OpenSearch 2.x ships dense-vector kNN via the `knn` query and the +`knn_vector` field type. + +```json +{ + "query": { + "knn": { + "embedding": { + "vector": "", + "k": 50 + } + } + } +} +``` + +Key knobs (cite [OpenSearch kNN](https://opensearch.org/docs/2.13/search-plugins/knn/knn-index/), accessed 2026-06-02): + +| Knob | Range | When to tune | +|---|---|---| +| `k` | int ≥ 1 | how many neighbours to return | +| `ef_search` (index-level) | int ≥ `k` | per-shard exploration depth — recall/latency knob (analog to ES `num_candidates`) | +| `space_type` (index-level) | `l2`, `cosinesimil`, `innerproduct`, `l1` | distance metric — fix at index time | + +### Hybrid (OpenSearch search-pipeline normalization processor) + +**OpenSearch does NOT ship the Elasticsearch `rrf` retriever.** Its +hybrid surface combines a lexical clause with a kNN clause via a +[search-pipeline normalization processor](https://opensearch.org/docs/2.13/search-plugins/search-pipelines/normalization-processor/) +that runs at search time: + +```json +{ + "query": { + "hybrid": { + "queries": [ + {"multi_match": {"query": "", "fields": ["title", "description"]}}, + {"knn": {"embedding": {"vector": "", "k": 50}}} + ] + } + } +} +``` + +Combined with a search pipeline whose normalization processor merges +the two result lists, e.g.: + +```json +{ + "phase_results_processors": [ + { + "normalization-processor": { + "normalization": {"technique": "min_max"}, + "combination": {"technique": "arithmetic_mean", "parameters": {"weights": [0.6, 0.4]}} + } + } + ] +} +``` + +**This is the OpenSearch-specific construct.** Elasticsearch's `rrf` +retriever (see +[`elasticsearch-tunable-params.md`](elasticsearch-tunable-params.md) +§"Hybrid") is NOT valid on OpenSearch — copying it produces a 400. The +two engines diverge here intentionally. + +Key knobs (cite [OpenSearch normalization processor](https://opensearch.org/docs/2.13/search-plugins/search-pipelines/normalization-processor/), accessed 2026-06-02): + +| Knob | Range | When to tune | +|---|---|---| +| `normalization.technique` | `min_max`, `l2` | how raw scores are scaled before combining | +| `combination.technique` | `arithmetic_mean`, `geometric_mean`, `harmonic_mean` | how normalized scores are combined | +| `combination.parameters.weights` | array of floats summing to 1.0 | per-clause weighting in the combined score | + +--- + +## See also + +- [`samples/templates/README.md`](../../samples/templates/README.md) — runnable ES/OS template library (engine-agnostic for the four lexical shapes) +- [`elasticsearch-tunable-params.md`](elasticsearch-tunable-params.md) — ES-specific reference (most lexical knobs are shared) +- [`solr-tunable-params.md`](solr-tunable-params.md) — Apache Solr cheatsheet +- [`docs/01_architecture/adapters.md`](../01_architecture/adapters.md) — `SearchAdapter` Protocol + unified-vocabulary cross-engine parameter map diff --git a/docs/06_vendor_docs/solr-tunable-params.md b/docs/06_vendor_docs/solr-tunable-params.md new file mode 100644 index 00000000..e0d6b41e --- /dev/null +++ b/docs/06_vendor_docs/solr-tunable-params.md @@ -0,0 +1,152 @@ + + +# Apache Solr tunable parameters + +Per-knob reference for the Apache Solr tunable parameters exposed by +RelyLoop's runnable [template library](../../samples/templates/solr/). +Solr's request shape is structurally different from Elasticsearch / +OpenSearch — flat request parameters (passed as URL query params or a +JSON body) rather than a nested query DSL — so this doc cites the +checked-in [`solr-9/`](solr-9/) and [`solr-10/`](solr-10/) Solr ref-guide +asciidoc source as the primary reference. + +**Engine versions covered:** Apache Solr **9.x** and **10.x** (the stock +`solr:9.x` and `solr:10.0` Docker images RelyLoop's compose stack uses). +The checked-in ref-guide pages under `solr-9/` and `solr-10/` are the +authoritative source — refresh from the upstream Apache Solr repo +(`raw.githubusercontent.com/apache/solr/releases/solr//solr/solr-ref-guide/modules/...`) +when a new Solr release lands. + +**Companion docs:** + +- [`adapters.md`](../01_architecture/adapters.md) §"Cross-engine parameter naming" +- [`samples/templates/solr/README.md`](../../samples/templates/solr/README.md) +- [`solr-9/`](solr-9/) / [`solr-10/`](solr-10/) — checked-in Apache Solr ref-guide asciidoc + +--- + +## Unified vocabulary (8 cross-engine params) + +The [`adapters.md`](../01_architecture/adapters.md) "Cross-engine +parameter naming" table includes the canonical unified-name → Solr- +native pivots. The `SolrAdapter.render()` method translates these +automatically, so a template may use either the unified key +(`field_boosts`) or the native key (`qf`) — both work. + +### `field_boosts` (per-field weights) + +- **Native Solr name:** `qf` (Query Fields), e.g. `qf=title^2.0 description^1.0 bullet_points^0.5`. +- **Range:** float ≥ 0 per field. Practical range 0.1–10. +- **When to tune:** identical reasoning to ES/OS — when one field carries more discriminative signal than the others. +- **Caveats:** Solr expects a space-joined string; the adapter does this pivot automatically. Order is preserved (Solr's scoring is order-independent but the wire form should match the template's intent). +- **Templates that use this param:** `edismax_basic.j2`, `boost_decay.j2` (via `title_boost`, `description_boost`, `bullet_points_boost`). +- **Source:** [`solr-10/`](solr-10/) — `dismax-query-parser` / `edismax-query-parser` modules (`qf` parameter). + +### `phrase_field_boosts` + +- **Native Solr name:** `pf` (Phrase Fields), `pf2` (2-gram), `pf3` (3-gram). Same space-joined `field^boost` syntax as `qf`. +- **Range:** float ≥ 0 per field. +- **When to tune:** when multi-token phrases occurring verbatim in a field should boost the document above documents that only match individual tokens. +- **Caveats:** `pf` is the canonical knob `ps` (phrase slop) acts on — without any `pf`, `ps` becomes a silent no-op. That's why `edismax_basic.j2` bakes `pf="title description"` even though it's not a declared tunable. +- **Templates that use this param:** `edismax_basic.j2` (baked-in literal — see the template body). + +### `tie_breaker` + +- **Native Solr name:** `tie` (the edismax/dismax tie-breaker). +- **Range:** float in `[0.0, 1.0]`. `0.0` = best-match-only; `1.0` = sum of all field scores. +- **When to tune:** same as ES — multi-field discrimination with semi-redundant fields. +- **Caveats:** the adapter accepts both `tie_breaker` (unified) and `tie` (native); the renderer pivots `tie_breaker` → `tie`. +- **Templates that use this param:** `edismax_basic.j2` (via `tie`). +- **Source:** [`solr-10/`](solr-10/) — `edismax-query-parser` module (`tie` parameter). + +### `min_should_match` + +- **Native Solr name:** `mm` (Minimum Match) — richer arithmetic syntax than ES (e.g. `2<-25% 9<-3`). +- **Range:** integer (`1`, `2`, …), percentage (`50%`, `75%`), or arithmetic syntax (`<-`). +- **When to tune:** balance recall vs. precision on multi-clause queries. +- **Caveats:** the arithmetic syntax only activates above the threshold clause count; on shorter queries Solr falls back to `100%`-equivalent. +- **Templates that use this param:** `edismax_basic.j2` (via `mm`). +- **Source:** [`solr-10/`](solr-10/) — `dismax-query-parser` module (`mm` parameter). + +### `fuzziness` + +- **Native Solr name:** none — Solr expresses fuzziness via the `~` operator inside the query parser (`title:laptop~2`), NOT a request parameter. +- **Range:** N/A (per-term `~N` edit-distance suffix in the query text itself). +- **Caveats:** the SolrAdapter rejects a `fuzziness` request-param with a targeted error message pointing operators at the `~` operator. (Verified by `backend/tests/unit/adapters/test_solr_render.py::TestRenderRejectsUnknownKeys::test_fuzziness_has_custom_message`.) +- **Templates that use this param:** none in the runnable Solr library (`edismax_basic.j2` and `boost_decay.j2` do not expose fuzziness). +- **Source:** [`solr-10/`](solr-10/) — `standard-query-parser` module ("Fuzzy Searches"). + +### `slop` + +- **Native Solr name:** `ps` (Phrase Slop) — applies to phrase queries generated from `pf` / `pf2` / `pf3`. +- **Range:** integer ≥ 0; practical 0–5. +- **When to tune:** when token-order should still matter on a phrase boost but exact order isn't strictly required. +- **Caveats:** `ps` is a no-op when no `pf` is set — that's why `edismax_basic.j2` bakes a `pf` literal. +- **Templates that use this param:** `edismax_basic.j2` (via `ps`). +- **Source:** [`solr-10/`](solr-10/) — `dismax-query-parser` module (`ps` parameter). + +### `boost_fn` (boost function) + +- **Native Solr name:** `bf` (Boost Function — additive) OR `boost` (multiplicative). The adapter picks one based on `boost_fn.combine="add"` or `"multiply"`. +- **Range:** any Solr function-query expression (`recip()`, `linear()`, `sum()`, `product()`, `scale()`, …). +- **When to tune:** any time a non-lexical signal (recency, popularity, price) should influence rank. +- **Caveats:** combining `bf` (additive) with `boost` (multiplicative) on the same query produces hard-to-reason-about scores; pick one combine semantics per template. +- **Templates that use this param:** `boost_decay.j2` (uses native `bf` with a `recip(ms(NOW, created_at), m, a, b)` expression interpolating `decay_scale` + `boost_weight`). +- **Source:** [`solr-10/`](solr-10/) — `dismax-query-parser` module (`bf` / `boost` parameters); function-query reference under `solr-ref-guide`. + +### `rerank_model` + +- **Native Solr name:** `rq={!ltr model= reRankDocs=}` (the LTR query parser, exposed when the `ltr` Solr module is loaded — RelyLoop's Compose stack loads it via `SOLR_MODULES=ltr`). +- **Range:** depends on the deployed LTR model (`reRankDocs` is the rescore window). +- **Caveats:** LTR model deployment is out of scope for the RelyLoop tuning loop (RelyLoop tunes query-time params, not model weights). See [`learning-to-rank.adoc`](solr-10/learning-to-rank.adoc) for the LTR module behaviour. +- **Templates that use this param:** none in the runnable Solr library. + +--- + +## Solr-specific knobs (template library coverage) + +### `boost_weight`, `decay_scale` + +- **Where used:** `boost_decay.j2` — interpolated into the rendered `bf` string `product(, recip(ms(NOW, created_at), , 1, 1))`. +- **Range:** + - `boost_weight` — float ≥ 0. The `product(...)` multiplier scaling a 0→1 `recip(x, m, 1, 1) = 1/(m*x + 1)` decay curve, so the MAXIMUM additive boost (at `x = 0`, i.e. age 0) is exactly `boost_weight`. (Interpolating `boost_weight` as both `a` and `b` of `recip` would instead cancel to 1.0 at age 0 and NOT scale the magnitude — that's why the template uses `product(...)`.) + - `decay_scale` — small positive number (the `m` slope; string-typed in the search space so scientific notation like `"3e-11"` interpolates cleanly into the rendered `bf` string). +- **When to tune:** when recency / age should boost lexical edismax matches. +- **Caveats:** the `recip` form requires `created_at` to be a date- or numeric-typed Solr field; on string-typed timestamps the function silently returns identical scores for every doc. +- **Source:** [`solr-10/`](solr-10/) — function-query reference (`recip`, `ms` function descriptions). + +### `tie`, `mm`, `ps` + +See the unified-vocabulary sections above (`tie_breaker`, +`min_should_match`, `slop`) — Solr exposes these as native short names. + +--- + +## Vector & hybrid — out of scope + +Solr 9+ ships dense-vector support via the +[`DenseVectorField`](solr-10/) type + the `{!knn}` query parser +(`q={!knn f=embedding topK=50}`). Solr also supports hybrid +search by composing `{!knn}` with `bq` / `bf` boost terms or via the +result-set merge primitives, but the wire shape is materially different +from ES's `rrf` retriever and from OpenSearch's normalization processor. + +**A Solr kNN / hybrid template is out of scope for this chore** — Solr's +dense-vector surface needs a separate field-type + (optionally) a +separate KNN configset, and is owned by a separate future effort, not +this content/docs chore. See `samples/templates/solr/README.md` "What's +NOT here (and why)" for the scoping decision. + +--- + +## See also + +- [`samples/templates/solr/README.md`](../../samples/templates/solr/README.md) — runnable Solr template library +- [`solr-9/`](solr-9/) / [`solr-10/`](solr-10/) — checked-in Apache Solr ref-guide asciidoc source +- [`elasticsearch-tunable-params.md`](elasticsearch-tunable-params.md) — Elasticsearch cheatsheet +- [`opensearch-tunable-params.md`](opensearch-tunable-params.md) — OpenSearch cheatsheet +- [`docs/01_architecture/adapters.md`](../01_architecture/adapters.md) — `SearchAdapter` Protocol + unified-vocabulary cross-engine parameter map diff --git a/docs/08_guides/tutorial-first-study.md b/docs/08_guides/tutorial-first-study.md index 2a950b61..c7d45386 100644 --- a/docs/08_guides/tutorial-first-study.md +++ b/docs/08_guides/tutorial-first-study.md @@ -461,7 +461,39 @@ your one decision.** --- -## Where to next +## Where to go next + +### Tune more than the demo template + +The tutorial registered `product_search.j2` — a deliberately minimal +demo template. RelyLoop ships a curated **runnable template library** +covering function-score decay, bool boosting, and phrase rescore on +ES/OpenSearch + edismax basic and recency-decay on Solr. Each library +template ships with a checked-in `.search_space.json` starter and a +copy-paste registration block. + +- [`samples/templates/README.md`](../../samples/templates/README.md) — + the four runnable ES/OpenSearch templates (`multi_match_basic`, + `function_score_decay`, `bool_boosted`, `rescore_phrase`) with + per-template "when to use", expected metric behavior, and a + copy-paste `curl` registration block per template. +- [`samples/templates/solr/README.md`](../../samples/templates/solr/README.md) — + the two runnable Solr templates (`edismax_basic`, `boost_decay`). + +### Look up a specific parameter + +Each tunable knob has a per-engine reference page with native + unified +names, valid ranges, "when to tune", caveats, and the templates that +declare it. + +- [`docs/06_vendor_docs/elasticsearch-tunable-params.md`](../06_vendor_docs/elasticsearch-tunable-params.md) +- [`docs/06_vendor_docs/opensearch-tunable-params.md`](../06_vendor_docs/opensearch-tunable-params.md) + (covers OpenSearch's hybrid normalization-processor — NOT the ES + `rrf` retriever) +- [`docs/06_vendor_docs/solr-tunable-params.md`](../06_vendor_docs/solr-tunable-params.md) + (grounded in the checked-in Solr 9 / 10 ref-guide source) + +### The rest of the project - The full feature set is in [`docs/02_product/mvp1-user-stories.md`](../02_product/mvp1-user-stories.md). - The architectural decisions are in diff --git a/samples/templates/README.md b/samples/templates/README.md index c5a05d48..d880a4a3 100644 --- a/samples/templates/README.md +++ b/samples/templates/README.md @@ -7,27 +7,41 @@ SPDX-License-Identifier: Apache-2.0 # Sample query templates This directory holds canonical Jinja2 query templates used by the -RelyLoop tutorial + demo seeding. Layout: +RelyLoop tutorial + demo seeding, plus the **runnable library** that +ships with the curated-template-library expansion (MVP2). Layout: ``` samples/templates/ - product_search.j2 # Elasticsearch / OpenSearch — the MVP1 demo - solr/ # Apache Solr templates (MVP2 — infra_adapter_solr) - products_edismax.j2 - products_dismax.j2 - products_lucene.j2 + product_search.j2 # ES / OpenSearch — the MVP1 demo + multi_match_basic.j2 # ES / OpenSearch — library: basic best_fields + multi_match_basic.search_space.json # starter SearchSpace for multi_match_basic + function_score_decay.j2 # ES / OpenSearch — library: function_score gauss decay + function_score_decay.search_space.json + bool_boosted.j2 # ES / OpenSearch — library: bool must/should/filter + bool_boosted.search_space.json + rescore_phrase.j2 # ES / OpenSearch — library: first-pass + phrase rescore + rescore_phrase.search_space.json + solr/ # Apache Solr templates (MVP2 — infra_adapter_solr) + products_edismax.j2 # demo + products_dismax.j2 # demo + products_lucene.j2 # demo + edismax_basic.j2 # library: edismax lexical + edismax_basic.search_space.json + boost_decay.j2 # library: edismax + recency boost + boost_decay.search_space.json + README.md # per-template Solr docs + registration blocks ``` ## Engine subdirectories -ES and OpenSearch share the same Query DSL surface — the MVP1 template at -the top level (`product_search.j2`) renders directly to an ES `multi_match` -body and works against both. Apache Solr's request shape is structurally -different (request parameters, not a query body), so Solr templates live -under `samples/templates/solr/` and render to a flat Solr-param dict. +ES and OpenSearch share the same Query DSL surface — top-level templates +(`product_search.j2` plus the four library templates) render directly to +an ES / OpenSearch query body and work against both engines. Apache +Solr's request shape is structurally different (request parameters, not +a query body), so Solr templates live under `samples/templates/solr/` +and render to a flat Solr-param dict. -Future engines (when/if they land) follow the same `/` subdir -convention. +Future engines follow the same `/` subdir convention. ## Template authoring rules @@ -36,15 +50,296 @@ convention. `query_templates` row that references it. 2. **Strict-undefined** — referencing an undeclared parameter raises `UndefinedError` at render time; declare every parameter the template - reads (`title_boost`, `field_boosts`, etc.) in the row's + reads (`title_boost`, `min_should_match`, …) in the row's `declared_params` map. -3. **JSON output** — the rendered output MUST parse as a JSON object. For - ES the object is the engine-native query body; for Solr the object - is a request-parameter dict whose keys are either Solr-native - (`defType`, `q`, `qf`, ...) or unified (`field_boosts`, `boost_fn`, ...) - per the [cross-engine parameter map](../../docs/01_architecture/adapters.md). -4. **No attribute access** — the Jinja sandbox forbids `.attr` access on - built-ins; flatten any nested param structures (`field_boosts` is a - flat dict, not `boost_config.fields`). +3. **JSON output** — the rendered output MUST parse as a JSON object. + For ES / OpenSearch the object is the engine-native query body; for + Solr the object is a request-parameter dict whose keys are either + Solr-native (`defType`, `q`, `qf`, …) or unified (`field_boosts`, + `boost_fn`, …) per the + [cross-engine parameter map](../../docs/01_architecture/adapters.md). +4. **No attribute access** — the Jinja sandbox forbids `.attr` access + on built-ins; flatten any nested param structures (`field_boosts` + is a flat dict, not `boost_config.fields`). +5. **`declared_params` keys MUST equal `.search_space.json` keys + exactly.** The platform validator `validate_against_template` + ([`backend/app/domain/study/search_space.py`](../../backend/app/domain/study/search_space.py)) + rejects both extra search-space keys and missing declared params. + Any structural value that should stay constant (field names, decay + function kind, `boost_mode`, `defType`) MUST be a Jinja **literal + baked into the template body**, NOT a declared param. +6. **Co-located `.search_space.json`.** Every runnable library template + ships a checked-in `.search_space.json` next to its `.j2`. + Cardinality (product of per-param bucket counts / categorical / + integer ranges) MUST stay under 10⁶; floats count as 100 buckets + each. With three floats already at the cap, additional tunable + knobs must be `categorical` or `int` with small ranges (see the + library spaces for the canonical pattern). +7. **Recommended registration name** — each runnable template carries a + recommended `--name` (e.g. `multi-match-basic-v1`) documented below. + The FR-7 client-side description map (if it ships) keys off this + name, so following the convention lets the Step-3 picker show a + "when to use" summary. See each engine subdir's `*.j2` files for canonical examples. + +--- + +## Runnable library templates + +The four ES/OpenSearch templates below are **engine-agnostic** — the +lexical / function-score / rescore DSL they emit is identical and valid +on both ES 8.11+ and OpenSearch 2.x. Because `query_templates.engine_type` +is a single value per row, an operator who runs both engines registers +the same body **once per engine** with a different `engine_type` value. +The `ENGINE_TYPE="elasticsearch" # or opensearch` variable in each +registration block below makes that explicit. + +### `multi_match_basic.j2` + +**When to use:** a fast lexical baseline you can drop into any catalog +that has `title` / `description` / `bullet_points` text fields. Best as +the "what do we beat?" starting point before reaching for decay, +boosting, or rescoring. + +**Declared params** (tunable): + +| Param | Type | Notes | +|---|---|---| +| `title_boost` | float | per-field boost on `title` | +| `description_boost` | float | per-field boost on `description` | +| `bullet_points_boost` | categorical | discrete float choices | +| `tie_breaker` | categorical | `best_fields` tie-breaker, range 0–1 | +| `fuzziness` | categorical | `"0"`, `"1"`, `"2"`, `"AUTO"` | + +`slop` is intentionally NOT exposed — it only affects `phrase` / +`phrase_prefix` multi_match, not `best_fields`. Phrase-slop tuning +lives on `rescore_phrase.j2`. + +**Expected metric behavior:** tuning `fuzziness` and `tie_breaker` +typically moves nDCG@10 by 1–3 points on noisy catalogs; on clean +catalogs the boosts dominate. Cardinality: 100 × 100 × 5 × 4 × 4 = 800,000. + +**Caveats:** `fuzziness="AUTO"` triggers per-term length-based fuzzy +expansion and is markedly slower on long queries than fixed edit- +distance choices. + +**Recommended registration name:** `multi-match-basic-v1`. + +**Register (copy-paste):** + +```bash +ENGINE_TYPE="elasticsearch" # or opensearch +jq -n \ + --arg body "$(cat samples/templates/multi_match_basic.j2)" \ + --arg engine "$ENGINE_TYPE" \ + '{ + name: "multi-match-basic-v1", + engine_type: $engine, + body: $body, + declared_params: { + title_boost: "float", + description_boost: "float", + bullet_points_boost: "categorical", + tie_breaker: "categorical", + fuzziness: "categorical" + } + }' \ +| curl -X POST http://localhost:8000/api/v1/query-templates \ + -H 'Content-Type: application/json' \ + --data-binary @- +``` + +### `function_score_decay.j2` + +**When to use:** when recency (or another numeric/date signal) should +boost lexical relevance. Pairs a `multi_match best_fields` first pass +with a Gaussian decay on `created_at`. + +**Declared params** (tunable): + +| Param | Type | Notes | +|---|---|---| +| `title_boost` | float | per-field boost on `title` | +| `description_boost` | float | per-field boost on `description` | +| `bullet_points_boost` | categorical | discrete float choices | +| `decay_scale` | categorical | gauss decay half-life: `30d`, `180d`, `365d` | +| `decay_offset` | categorical | plateau before decay: `0d`, `30d` | +| `decay_decay` | categorical | value at `decay_scale` distance: `0.3`, `0.5`, `0.7` | + +The decay field name (`created_at`), the decay function kind (`gauss`), +and `boost_mode` (`multiply`) are baked into the template as Jinja +literals — they're structural decisions, not tunable knobs. + +**Expected metric behavior:** large nDCG@10 deltas on news / catalog +data with strong recency bias; little to no signal on time-invariant +catalogs. Cardinality: 100 × 100 × 3 × 3 × 2 × 3 = 540,000. + +**Caveats:** the `gauss` decay requires `created_at` to be a date- or +numeric-mapped field; on string-typed timestamps it silently scores +all docs equally. Verify the mapping before registering. + +**Recommended registration name:** `function-score-decay-v1`. + +**Register (copy-paste):** + +```bash +ENGINE_TYPE="elasticsearch" # or opensearch +jq -n \ + --arg body "$(cat samples/templates/function_score_decay.j2)" \ + --arg engine "$ENGINE_TYPE" \ + '{ + name: "function-score-decay-v1", + engine_type: $engine, + body: $body, + declared_params: { + title_boost: "float", + description_boost: "float", + bullet_points_boost: "categorical", + decay_scale: "categorical", + decay_offset: "categorical", + decay_decay: "categorical" + } + }' \ +| curl -X POST http://localhost:8000/api/v1/query-templates \ + -H 'Content-Type: application/json' \ + --data-binary @- +``` + +### `bool_boosted.j2` + +**When to use:** when you need explicit clause-level control +(`must` for recall, `should` for ranking signal, `minimum_should_match` +for precision tuning) and want to optimize `min_should_match` directly. + +**Declared params** (tunable): + +| Param | Type | Notes | +|---|---|---| +| `title_boost` | float | should-clause boost on `title` | +| `description_boost` | float | should-clause boost on `description` | +| `bullet_points_boost` | categorical | should-clause boost on `bullet_points` | +| `min_should_match` | categorical | Elasticsearch `minimum_should_match` syntax: `1`, `2`, `50%`, `75%`, `2<-25% 9<-3` | + +The `must`-clause field (`title`), the should-clause field names +(`title`, `description`, `bullet_points`), and the `filter` clause (an +`exists` filter on `title` — a structural floor keeping untitled docs +out of results; replace with your own term/range filters) are baked-in +literals. + +**Expected metric behavior:** tightening `min_should_match` (e.g. from +`50%` → `75%`) typically lifts precision @ low N at the cost of recall; +the optimizer trades these off against the boost weights. +Cardinality: 100 × 100 × 5 × 5 = 250,000. + +**Caveats:** the arithmetic-syntax choice `2<-25% 9<-3` only kicks in on +queries with ≥ 3 should-clauses; on shorter queries it falls back to +`100%`-must behaviour. Combine with longer query sets to see the +trade-off in metrics. + +**Recommended registration name:** `bool-boosted-v1`. + +**Register (copy-paste):** + +```bash +ENGINE_TYPE="elasticsearch" # or opensearch +jq -n \ + --arg body "$(cat samples/templates/bool_boosted.j2)" \ + --arg engine "$ENGINE_TYPE" \ + '{ + name: "bool-boosted-v1", + engine_type: $engine, + body: $body, + declared_params: { + title_boost: "float", + description_boost: "float", + bullet_points_boost: "categorical", + min_should_match: "categorical" + } + }' \ +| curl -X POST http://localhost:8000/api/v1/query-templates \ + -H 'Content-Type: application/json' \ + --data-binary @- +``` + +### `rescore_phrase.j2` + +**When to use:** when a fast lexical first pass gets recall right but +exact-phrase matches should be promoted within the top-N. Combines +`multi_match best_fields` with a `match_phrase` rescore over a tunable +window. + +**Declared params** (tunable): + +| Param | Type | Notes | +|---|---|---| +| `title_boost` | categorical | first-pass boost on `title` | +| `description_boost` | categorical | first-pass boost on `description` | +| `bullet_points_boost` | categorical | first-pass boost on `bullet_points` | +| `rescore_window_size` | categorical | hits to rescore: `10`, `25`, `50`, `100`, `200` | +| `rescore_query_weight` | categorical | first-pass weight: `0.5`, `1.0`, `1.5`, `2.0` | +| `rescore_phrase_slop` | int | phrase-slop tolerance (0–5) on the rescore pass | + +First-pass `type: best_fields` and the rescore phrase field (`title`) +are baked-in literals. + +**Expected metric behavior:** large MRR / nDCG@10 lifts on catalogs +where multi-word product names appear verbatim in `title`; minimal +signal on free-text fields. Cardinality: 5 × 5 × 5 × 5 × 4 × 6 = 75,000. + +**Caveats:** rescore is a per-shard operation, so `window_size` larger +than the first-pass `from + size` is wasted work. Keep `window_size` +≤ 200 for catalog-search workloads. + +**Recommended registration name:** `rescore-phrase-v1`. + +**Register (copy-paste):** + +```bash +ENGINE_TYPE="elasticsearch" # or opensearch +jq -n \ + --arg body "$(cat samples/templates/rescore_phrase.j2)" \ + --arg engine "$ENGINE_TYPE" \ + '{ + name: "rescore-phrase-v1", + engine_type: $engine, + body: $body, + declared_params: { + title_boost: "categorical", + description_boost: "categorical", + bullet_points_boost: "categorical", + rescore_window_size: "categorical", + rescore_query_weight: "categorical", + rescore_phrase_slop: "int" + } + }' \ +| curl -X POST http://localhost:8000/api/v1/query-templates \ + -H 'Content-Type: application/json' \ + --data-binary @- +``` + +--- + +## What's NOT here (and why) + +- **`knn_only.j2` / `hybrid_rrf.j2`** — neither ships as a runnable + template. The trial-runner render context is exactly + `{**params, "query_text": query_text}` — there is no query-vector + injection mechanism, so a pure-kNN or hybrid template cannot render + a complete request today. Instead, engine-correct reference snippets + live in the ES and OpenSearch tunable-params cheatsheets — see + [`docs/06_vendor_docs/elasticsearch-tunable-params.md`](../../docs/06_vendor_docs/elasticsearch-tunable-params.md) + and + [`docs/06_vendor_docs/opensearch-tunable-params.md`](../../docs/06_vendor_docs/opensearch-tunable-params.md). +- **A Solr kNN / hybrid template** — Solr's dense-vector and hybrid + surface is materially different from ES / OpenSearch and is owned + by a separate future effort. The Solr cheatsheet documents the + out-of-scope rationale. + +## See also + +- [`docs/06_vendor_docs/elasticsearch-tunable-params.md`](../../docs/06_vendor_docs/elasticsearch-tunable-params.md) — per-knob native names, ranges, citations, and template back-links +- [`docs/06_vendor_docs/opensearch-tunable-params.md`](../../docs/06_vendor_docs/opensearch-tunable-params.md) +- [`docs/06_vendor_docs/solr-tunable-params.md`](../../docs/06_vendor_docs/solr-tunable-params.md) +- [`docs/01_architecture/adapters.md`](../../docs/01_architecture/adapters.md) — `SearchAdapter` Protocol + unified-vocabulary cross-engine parameter map +- [`docs/08_guides/tutorial-first-study.md`](../../docs/08_guides/tutorial-first-study.md) — end-to-end first study walkthrough diff --git a/samples/templates/bool_boosted.j2 b/samples/templates/bool_boosted.j2 new file mode 100644 index 00000000..199c528f --- /dev/null +++ b/samples/templates/bool_boosted.j2 @@ -0,0 +1,47 @@ +{#- + bool_boosted.j2 — bool query with must / should / filter clauses and a + tunable `minimum_should_match`. + + Declared (tunable) params: + title_boost float per-field boost on `title` (should) + description_boost float per-field boost on `description` (should) + bullet_points_boost float per-field boost on `bullet_points` (should) + min_should_match string categorical — Elasticsearch's + minimum_should_match syntax + (e.g. "1", "2", "75%", "2<-25% 9<-3") + + Baked-in literals (NOT declared / NOT tunable): + must clause field "title" (match) — drives recall + should clauses' field names "title", "description", "bullet_points" + filter clause an `exists` filter on `title` — a no-op-ish + structural floor that keeps untitled docs + out of the result set. Baked in (not tunable) + so the must/should/filter shape FR-1 names + is structurally present; operators replace it + with their own term/range filters as needed. + + `query_text` is rendered through Jinja's `tojson` filter so a query with + a double-quote or newline still yields valid JSON. + + Compatible with ES 8.11+ and OpenSearch 2.x. + + Recommended registration name: `bool-boosted-v1`. +-#} +{ + "query": { + "bool": { + "must": [ + {"match": {"title": {{ query_text | tojson }}}} + ], + "should": [ + {"match": {"title": {"query": {{ query_text | tojson }}, "boost": {{ title_boost }}}}}, + {"match": {"description": {"query": {{ query_text | tojson }}, "boost": {{ description_boost }}}}}, + {"match": {"bullet_points": {"query": {{ query_text | tojson }}, "boost": {{ bullet_points_boost }}}}} + ], + "filter": [ + {"exists": {"field": "title"}} + ], + "minimum_should_match": "{{ min_should_match }}" + } + } +} diff --git a/samples/templates/bool_boosted.search_space.json b/samples/templates/bool_boosted.search_space.json new file mode 100644 index 00000000..edfd492a --- /dev/null +++ b/samples/templates/bool_boosted.search_space.json @@ -0,0 +1,8 @@ +{ + "params": { + "title_boost": {"type": "float", "low": 0.5, "high": 10.0}, + "description_boost": {"type": "float", "low": 0.5, "high": 10.0}, + "bullet_points_boost": {"type": "categorical", "choices": [0.5, 1.0, 2.0, 5.0, 10.0]}, + "min_should_match": {"type": "categorical", "choices": ["1", "2", "50%", "75%", "2<-25% 9<-3"]} + } +} diff --git a/samples/templates/function_score_decay.j2 b/samples/templates/function_score_decay.j2 new file mode 100644 index 00000000..90fb3a4d --- /dev/null +++ b/samples/templates/function_score_decay.j2 @@ -0,0 +1,55 @@ +{#- + function_score_decay.j2 — function_score with a Gaussian decay over a + numeric/date field, combined with a lexical multi_match (best_fields). + + Declared (tunable) params: + title_boost float per-field boost on `title` + description_boost float per-field boost on `description` + bullet_points_boost float per-field boost on `bullet_points` + decay_scale string e.g. "30d" or "180d" — controls the + half-life of the gauss decay + decay_offset string e.g. "0d" or "7d" — the plateau + before decay starts + decay_decay float value the function returns at + `decay_scale` distance (0–1) + + Baked-in literals (NOT declared / NOT tunable): + decay field name "created_at" + decay function kind "gauss" + boost_mode "multiply" + + Compatible with ES 8.11+ and OpenSearch 2.x. `decay_scale` / `decay_offset` + are strings to support both numeric (`"30"`) and date-math (`"30d"`) + syntaxes; the search space below picks date-math choices. + + Recommended registration name: `function-score-decay-v1`. +-#} +{ + "query": { + "function_score": { + "query": { + "multi_match": { + "query": {{ query_text | tojson }}, + "type": "best_fields", + "fields": [ + "title^{{ title_boost }}", + "description^{{ description_boost }}", + "bullet_points^{{ bullet_points_boost }}" + ] + } + }, + "functions": [ + { + "gauss": { + "created_at": { + "scale": "{{ decay_scale }}", + "offset": "{{ decay_offset }}", + "decay": {{ decay_decay }} + } + } + } + ], + "boost_mode": "multiply" + } + } +} diff --git a/samples/templates/function_score_decay.search_space.json b/samples/templates/function_score_decay.search_space.json new file mode 100644 index 00000000..f0488f42 --- /dev/null +++ b/samples/templates/function_score_decay.search_space.json @@ -0,0 +1,10 @@ +{ + "params": { + "title_boost": {"type": "float", "low": 0.5, "high": 10.0}, + "description_boost": {"type": "float", "low": 0.5, "high": 10.0}, + "bullet_points_boost": {"type": "categorical", "choices": [0.5, 1.0, 2.0]}, + "decay_scale": {"type": "categorical", "choices": ["30d", "180d", "365d"]}, + "decay_offset": {"type": "categorical", "choices": ["0d", "30d"]}, + "decay_decay": {"type": "categorical", "choices": [0.3, 0.5, 0.7]} + } +} diff --git a/samples/templates/multi_match_basic.j2 b/samples/templates/multi_match_basic.j2 new file mode 100644 index 00000000..4169d736 --- /dev/null +++ b/samples/templates/multi_match_basic.j2 @@ -0,0 +1,43 @@ +{#- + multi_match_basic.j2 — basic lexical multi_match (best_fields) for ES + OpenSearch. + + Declared (tunable) params: + title_boost float per-field boost on `title` + description_boost float per-field boost on `description` + bullet_points_boost float per-field boost on `bullet_points` + tie_breaker float range 0.0–1.0 (best_fields tie-breaker) + fuzziness string one of "0", "1", "2", "AUTO" + + Baked-in literals (NOT declared / NOT tunable): + field list ["title", "description", "bullet_points"] + type "best_fields" + + `slop` is intentionally NOT exposed here — it only affects `phrase` / + `phrase_prefix` multi_match, not `best_fields`. Phrase slop lives on + `rescore_phrase.j2` where it is valid (spec §7 FR-1 / cycle 3, GPT-5.5 F1). + + `query_text` is rendered through Jinja's `tojson` filter so a query + containing a double-quote or newline still produces valid JSON (the + filter emits the surrounding quotes — no manual `"..."` wrapping). + + Renders to a native ES/OpenSearch query body. Compatible with ES 8.11+ + and OpenSearch 2.x (lexical DSL is identical across both engines). + + Recommended registration name: `multi-match-basic-v1` + (see samples/templates/README.md "Runnable library templates"). +-#} +{ + "query": { + "multi_match": { + "query": {{ query_text | tojson }}, + "type": "best_fields", + "tie_breaker": {{ tie_breaker }}, + "fuzziness": "{{ fuzziness }}", + "fields": [ + "title^{{ title_boost }}", + "description^{{ description_boost }}", + "bullet_points^{{ bullet_points_boost }}" + ] + } + } +} diff --git a/samples/templates/multi_match_basic.search_space.json b/samples/templates/multi_match_basic.search_space.json new file mode 100644 index 00000000..4b5bd139 --- /dev/null +++ b/samples/templates/multi_match_basic.search_space.json @@ -0,0 +1,9 @@ +{ + "params": { + "title_boost": {"type": "float", "low": 0.5, "high": 10.0}, + "description_boost": {"type": "float", "low": 0.5, "high": 10.0}, + "bullet_points_boost": {"type": "categorical", "choices": [0.5, 1.0, 2.0, 5.0, 10.0]}, + "tie_breaker": {"type": "categorical", "choices": [0.0, 0.3, 0.7, 1.0]}, + "fuzziness": {"type": "categorical", "choices": ["0", "1", "2", "AUTO"]} + } +} diff --git a/samples/templates/rescore_phrase.j2 b/samples/templates/rescore_phrase.j2 new file mode 100644 index 00000000..26221acf --- /dev/null +++ b/samples/templates/rescore_phrase.j2 @@ -0,0 +1,52 @@ +{#- + rescore_phrase.j2 — first-pass lexical (multi_match best_fields) + second- + pass phrase rescore over the top-N first-pass hits. + + Declared (tunable) params: + title_boost float per-field boost on `title` (first pass) + description_boost float per-field boost on `description` (first pass) + bullet_points_boost float per-field boost on `bullet_points` (first pass) + rescore_window_size int how many first-pass hits to rescore + rescore_query_weight float weight of the original first-pass score + rescore_phrase_slop int phrase-slop tolerated on the rescore pass + + Baked-in literals (NOT declared / NOT tunable): + first-pass type "best_fields" + rescore phrase field "title" + + `query_text` is rendered through Jinja's `tojson` filter so a query with + a double-quote or newline still yields valid JSON. + + Compatible with ES 8.11+ and OpenSearch 2.x. (`phrase_slop` is valid here + because the rescore query is a `match_phrase` — unlike best_fields where + slop is a silent no-op.) + + Recommended registration name: `rescore-phrase-v1`. +-#} +{ + "query": { + "multi_match": { + "query": {{ query_text | tojson }}, + "type": "best_fields", + "fields": [ + "title^{{ title_boost }}", + "description^{{ description_boost }}", + "bullet_points^{{ bullet_points_boost }}" + ] + } + }, + "rescore": { + "window_size": {{ rescore_window_size }}, + "query": { + "rescore_query": { + "match_phrase": { + "title": { + "query": {{ query_text | tojson }}, + "slop": {{ rescore_phrase_slop }} + } + } + }, + "query_weight": {{ rescore_query_weight }} + } + } +} diff --git a/samples/templates/rescore_phrase.search_space.json b/samples/templates/rescore_phrase.search_space.json new file mode 100644 index 00000000..27de678a --- /dev/null +++ b/samples/templates/rescore_phrase.search_space.json @@ -0,0 +1,10 @@ +{ + "params": { + "title_boost": {"type": "categorical", "choices": [0.5, 1.0, 2.0, 5.0, 10.0]}, + "description_boost": {"type": "categorical", "choices": [0.5, 1.0, 2.0, 5.0, 10.0]}, + "bullet_points_boost": {"type": "categorical", "choices": [0.5, 1.0, 2.0, 5.0, 10.0]}, + "rescore_window_size": {"type": "categorical", "choices": [10, 25, 50, 100, 200]}, + "rescore_query_weight": {"type": "categorical", "choices": [0.5, 1.0, 1.5, 2.0]}, + "rescore_phrase_slop": {"type": "int", "low": 0, "high": 5} + } +} diff --git a/samples/templates/solr/README.md b/samples/templates/solr/README.md new file mode 100644 index 00000000..40cd5dc8 --- /dev/null +++ b/samples/templates/solr/README.md @@ -0,0 +1,159 @@ + + +# Solr query templates + +Apache Solr templates (Solr 9.x + 10.x). Renders to a flat Solr request- +parameter dict per the `SolrAdapter.render()` contract — mixing Solr-native +keys (`defType`, `q`, `qf`, `pf`, `tie`, `mm`, `ps`, `bf`, `boost`, `fl`, +…) with unified-pivot keys (`field_boosts` → `qf`, `tie_breaker` → `tie`, +`min_should_match` → `mm`, `slop` → `ps`, `boost_fn` → `bf` / `boost`) +documented in +[`docs/01_architecture/adapters.md`](../../../docs/01_architecture/adapters.md). + +Layout: + +``` +samples/templates/solr/ + products_edismax.j2 # demo (infra_adapter_solr) + products_dismax.j2 # demo + products_lucene.j2 # demo + edismax_basic.j2 # library: edismax lexical + edismax_basic.search_space.json + boost_decay.j2 # library: edismax + recency boost + boost_decay.search_space.json +``` + +The demo templates (`products_*.j2`) are byte-stable — they're read by +`backend/app/services/demo_seeding.py` and the smoke path. The library +templates below are NEW additions an operator registers on demand. + +## Runnable library templates + +### `edismax_basic.j2` + +**When to use:** the canonical Solr lexical baseline — wider tunable +surface than the demo `products_edismax.j2`. Drop in when you want to +optimize `tie` / `mm` / `ps` along with per-field boosts. + +**Declared params** (tunable): + +| Param | Type | Notes | +|---|---|---| +| `title_boost` | float | per-field boost on `title` | +| `description_boost` | categorical | discrete float choices | +| `bullet_points_boost` | categorical | discrete float choices | +| `tie` | categorical | edismax tie-breaker, `0.0`–`1.0` | +| `mm` | categorical | Solr `mm` arithmetic syntax: `1`, `2`, `75%`, `2<-25% 9<-3` | +| `ps` | int | phrase-slop tolerance (0–3) | + +Baked into the body: `defType=edismax`, the qf field names, the +`pf="title description"` phrase-fields source (without `pf`, `ps` is a +silent no-op — Gemini finding on the spec, accepted), and `fl=*,score`. + +**Expected metric behavior:** larger `tie` boosts long-tail matches at +the cost of headline relevance; `ps` lift is workload-dependent. +Cardinality: 100 × 5 × 5 × 5 × 4 × 4 = 200,000. + +**Caveats:** `mm`'s arithmetic syntax (`2<-25% 9<-3`) only kicks in on +queries with ≥ 3 clauses; on short queries Solr falls back to `100%`. + +**Recommended registration name:** `edismax-basic-v1`. + +**Register (copy-paste):** + +```bash +jq -n \ + --arg body "$(cat samples/templates/solr/edismax_basic.j2)" \ + '{ + name: "edismax-basic-v1", + engine_type: "solr", + body: $body, + declared_params: { + title_boost: "float", + description_boost: "categorical", + bullet_points_boost: "categorical", + tie: "categorical", + mm: "categorical", + ps: "int" + } + }' \ +| curl -X POST http://localhost:8000/api/v1/query-templates \ + -H 'Content-Type: application/json' \ + --data-binary @- +``` + +### `boost_decay.j2` + +**When to use:** when recency (or another date-typed signal) should +boost lexical edismax matches. Uses +`product(boost_weight, recip(ms(NOW, created_at), decay_scale, 1, 1))` +as an additive boost via `bf` — a 0→1 recip() decay curve scaled by +`boost_weight`, so the max additive boost (at age 0) equals `boost_weight`. + +**Declared params** (tunable), exact set: + +| Param | Type | Notes | +|---|---|---| +| `title_boost` | float | per-field boost on `title` | +| `description_boost` | float | per-field boost on `description` | +| `bullet_points_boost` | categorical | discrete float choices | +| `boost_weight` | categorical | max additive boost magnitude (at age 0) — the `product(...)` multiplier | +| `decay_scale` | categorical | recip slope (`m`), smaller = slower decay | + +The `bf` function-expression skeleton, the decay field name +(`created_at`), and `defType=edismax` are baked-in literals. `boost_weight` +and `decay_scale` are interpolated into the rendered `bf` string at +render time. + +**Expected metric behavior:** large nDCG@10 deltas on catalogs with +strong recency bias; little signal on time-invariant content. +Cardinality: 100 × 100 × 5 × 4 × 3 = 600,000. + +**Caveats:** the `recip(ms(NOW,…), …)` form requires `created_at` to +be a date- or numeric-typed Solr field; on string-typed timestamps the +function silently returns identical scores. Verify the schema before +registering. + +**Recommended registration name:** `boost-decay-v1`. + +**Register (copy-paste):** + +```bash +jq -n \ + --arg body "$(cat samples/templates/solr/boost_decay.j2)" \ + '{ + name: "boost-decay-v1", + engine_type: "solr", + body: $body, + declared_params: { + title_boost: "float", + description_boost: "float", + bullet_points_boost: "categorical", + boost_weight: "categorical", + decay_scale: "categorical" + } + }' \ +| curl -X POST http://localhost:8000/api/v1/query-templates \ + -H 'Content-Type: application/json' \ + --data-binary @- +``` + +## What's NOT here (and why) + +- **A Solr kNN template** (e.g. `{!knn}`) — Solr's dense-vector surface + has materially different infra requirements (a `vector` field type, + potentially a separate KNN configset) and is owned by a separate + future effort, not this content/docs chore. +- **A Solr hybrid template** — same reason. The + [`solr-tunable-params.md`](../../../docs/06_vendor_docs/solr-tunable-params.md) + cheatsheet documents the out-of-scope rationale. + +## See also + +- [`docs/06_vendor_docs/solr-tunable-params.md`](../../../docs/06_vendor_docs/solr-tunable-params.md) — per-knob native names, ranges, citations, and template back-links +- [`docs/06_vendor_docs/solr-9/`](../../../docs/06_vendor_docs/solr-9/) / [`solr-10/`](../../../docs/06_vendor_docs/solr-10/) — Solr ref-guide asciidoc source +- [`docs/01_architecture/adapters.md`](../../../docs/01_architecture/adapters.md) — `SearchAdapter` Protocol + cross-engine parameter map diff --git a/samples/templates/solr/boost_decay.j2 b/samples/templates/solr/boost_decay.j2 new file mode 100644 index 00000000..424560d3 --- /dev/null +++ b/samples/templates/solr/boost_decay.j2 @@ -0,0 +1,44 @@ +{#- + boost_decay.j2 — Apache Solr edismax with a recency/proximity boost + function. Combines lexical edismax with a `recip(ms(NOW,created_at),...)` + decay applied as an additive boost (Solr's `bf` semantics). + + Declared (tunable) params, EXACT set: + title_boost float per-field boost on `title` + description_boost float per-field boost on `description` + bullet_points_boost categorical discrete float choices + boost_weight categorical overall additive boost strength + decay_scale categorical recip slope (smaller = slower decay) + + Baked-in literals (NOT declared / NOT tunable): + defType "edismax" + qf field names "title", "description", "bullet_points" + (via field_boosts → qf pivot) + bf function skeleton "recip(ms(NOW,created_at), , + , )" + decay field name "created_at" + fl "*,score" + + Solr's `recip(x, m, a, b) = a / (m*x + b)`. With `a = b = 1` and + `x = ms(NOW, created_at)`: + age=0 → recip(...) = 1/1 = 1.0 (max) + age large → recip(...) → 0 + `product(boost_weight, recip(...))` then scales that 0→1 decay curve by + `boost_weight`, so the MAXIMUM additive boost (at age 0) is exactly + `boost_weight` — the knob controls boost magnitude as documented, instead + of cancelling out at age 0 (which is what `recip(...,boost_weight,boost_weight)` + would do). Smaller `decay_scale` (the `m` slope) = slower decay. + + Recommended registration name: `boost-decay-v1`. +-#} +{ + "defType": "edismax", + "q": {{ query_text | tojson }}, + "field_boosts": { + "title": {{ title_boost }}, + "description": {{ description_boost }}, + "bullet_points": {{ bullet_points_boost }} + }, + "bf": "product({{ boost_weight }},recip(ms(NOW,created_at),{{ decay_scale }},1,1))", + "fl": "*,score" +} diff --git a/samples/templates/solr/boost_decay.search_space.json b/samples/templates/solr/boost_decay.search_space.json new file mode 100644 index 00000000..a44a760e --- /dev/null +++ b/samples/templates/solr/boost_decay.search_space.json @@ -0,0 +1,9 @@ +{ + "params": { + "title_boost": {"type": "float", "low": 0.5, "high": 10.0}, + "description_boost": {"type": "float", "low": 0.5, "high": 10.0}, + "bullet_points_boost": {"type": "categorical", "choices": [0.5, 1.0, 2.0, 5.0, 10.0]}, + "boost_weight": {"type": "categorical", "choices": [0.5, 1.0, 2.0, 5.0]}, + "decay_scale": {"type": "categorical", "choices": ["1e-11", "3e-11", "1e-10"]} + } +} diff --git a/samples/templates/solr/edismax_basic.j2 b/samples/templates/solr/edismax_basic.j2 new file mode 100644 index 00000000..7fcff5a0 --- /dev/null +++ b/samples/templates/solr/edismax_basic.j2 @@ -0,0 +1,44 @@ +{#- + edismax_basic.j2 — Apache Solr edismax lexical template (library shape, + wider tunable surface than the demo ``products_edismax.j2``). + + Declared (tunable) params: + title_boost float per-field boost on `title` + description_boost categorical discrete float choices for description + bullet_points_boost categorical discrete float choices for bullet_points + tie categorical edismax tie-breaker, 0.0–1.0 + mm categorical Solr `mm` arithmetic syntax + ps int phrase-slop tolerance (0–3) + + Baked-in literals (NOT declared / NOT tunable): + defType "edismax" + qf field names "title", "description", "bullet_points" + (via field_boosts → qf pivot) + pf "title description" — Solr's phrase-fields + param. Baked-in because the declared-tunable `ps` (phrase slop) ONLY + affects phrase queries generated from `pf`; without any `pf` the `ps` + knob would be a silent no-op (Gemini finding on PR #413 spec — accepted). + fl "*,score" + + Output dict mixes Solr-native + unified-pivot keys: + defType / q / pf / ps / fl — Solr-native (passed through unchanged). + field_boosts — pivoted to qf by SolrAdapter.render. + tie_breaker — pivoted to tie. + min_should_match — pivoted to mm. + + Recommended registration name: `edismax-basic-v1`. +-#} +{ + "defType": "edismax", + "q": {{ query_text | tojson }}, + "field_boosts": { + "title": {{ title_boost }}, + "description": {{ description_boost }}, + "bullet_points": {{ bullet_points_boost }} + }, + "pf": "title description", + "tie_breaker": {{ tie }}, + "min_should_match": "{{ mm }}", + "slop": {{ ps }}, + "fl": "*,score" +} diff --git a/samples/templates/solr/edismax_basic.search_space.json b/samples/templates/solr/edismax_basic.search_space.json new file mode 100644 index 00000000..a33cc987 --- /dev/null +++ b/samples/templates/solr/edismax_basic.search_space.json @@ -0,0 +1,10 @@ +{ + "params": { + "title_boost": {"type": "float", "low": 0.5, "high": 10.0}, + "description_boost": {"type": "categorical", "choices": [0.5, 1.0, 2.0, 5.0, 10.0]}, + "bullet_points_boost": {"type": "categorical", "choices": [0.5, 1.0, 2.0, 5.0, 10.0]}, + "tie": {"type": "categorical", "choices": [0.0, 0.3, 0.5, 0.7, 1.0]}, + "mm": {"type": "categorical", "choices": ["1", "2", "75%", "2<-25% 9<-3"]}, + "ps": {"type": "int", "low": 0, "high": 3} + } +} diff --git a/state.md b/state.md index 69125e11..fd05bffa 100644 --- a/state.md +++ b/state.md @@ -14,8 +14,8 @@ MVP1 (v0.1) **shipped** — all six differentiators live (Bayesian/TPE optimizer ## Current branch / execution context -- **Branch:** `main` (PR #367 `infra_solr_ci_readiness` Phase 1 just merged, `214cdfcd`). The `pr.yml` backend job is green again; the `smoke` job stays red on the deferred Phase-2 Solr-container crash. -- **Active feature:** _None in flight._ `infra_solr_smoke_stability` shipped 2026-06-02 (PR #383, squash-merged `d32b9714`); Phase 1 sibling `infra_solr_ci_readiness` shipped 2026-06-01 (PR #367). Diagnostics + Lever 0 (perms) + Lever 1 (heap-cap) all in place; the remaining `smoke` redness is the Playwright reseed runtime budget (captured as [`infra_smoke_reseed_runtime_budget`](docs/00_overview/planned_features/02_mvp2/infra_smoke_reseed_runtime_budget/idea.md) — P1, three candidate fixes documented with Option A as default). Next: either pull that follow-up next, or pull from the MVP2 Idea backlog (run `/pipeline status`). +- **Branch:** `chore/template-library-expansion` (PR open). `chore_template_library_expansion` implementation pass: 6 runnable Jinja templates (4 ES/OS + 2 Solr) with co-located `.search_space.json` starters; three per-engine tunable-params cheatsheets (`elasticsearch-`, `opensearch-`, `solr-tunable-params.md`) with kNN + hybrid reference snippets (ES `rrf` retriever vs OpenSearch normalization-processor — engine-correct on both sides); vendor-docs README index rows; tutorial "Where to go next" library + cheatsheet links; FR-7 **shipped** client-side via `ui/src/lib/template-descriptions.ts` (description map + `cheatsheetUrlFor` resolver) + optional `learnMoreHref` prop on `InfoTooltip` (additive, no shared-tooltip refactor) + Step-3 modal summary wiring. No source under `backend/app/`; no migration (Alembic head stays at `0022`). +- **Active feature:** `chore_template_library_expansion` in PR review. `infra_solr_smoke_stability` shipped 2026-06-02 (PR #383, squash-merged `d32b9714`); Phase 1 sibling `infra_solr_ci_readiness` shipped 2026-06-01 (PR #367). Diagnostics + Lever 0 (perms) + Lever 1 (heap-cap) all in place; the remaining `smoke` redness is the Playwright reseed runtime budget (captured as [`infra_smoke_reseed_runtime_budget`](docs/00_overview/planned_features/02_mvp2/infra_smoke_reseed_runtime_budget/idea.md) — P1, three candidate fixes documented with Option A as default). - **Alembic head:** `0022_solr_engine_auth_check` (added by `infra_adapter_solr` Story A6 — extends `clusters.engine_type` + `clusters.auth_kind` CHECK constraints for Solr). - **Python:** 3.13. **Frontend stack:** Next 16 (App Router + Turbopack), React 19, Tailwind 4 (CSS-first), Vitest 4, ESLint 9 (flat), TypeScript 6, Playwright (chromium, single worker) for E2E. - **Coverage gates:** backend 80% (`fail_under` in pyproject), UI vitest + tsc + ESLint + Next build, plus a full-stack smoke E2E job. Live pass counts: see the latest `pr.yml` run (the historical per-feature counts moved to `state_history.md`). diff --git a/ui/public/docs/tutorial-first-study.md b/ui/public/docs/tutorial-first-study.md index 2a950b61..c7d45386 100644 --- a/ui/public/docs/tutorial-first-study.md +++ b/ui/public/docs/tutorial-first-study.md @@ -461,7 +461,39 @@ your one decision.** --- -## Where to next +## Where to go next + +### Tune more than the demo template + +The tutorial registered `product_search.j2` — a deliberately minimal +demo template. RelyLoop ships a curated **runnable template library** +covering function-score decay, bool boosting, and phrase rescore on +ES/OpenSearch + edismax basic and recency-decay on Solr. Each library +template ships with a checked-in `.search_space.json` starter and a +copy-paste registration block. + +- [`samples/templates/README.md`](../../samples/templates/README.md) — + the four runnable ES/OpenSearch templates (`multi_match_basic`, + `function_score_decay`, `bool_boosted`, `rescore_phrase`) with + per-template "when to use", expected metric behavior, and a + copy-paste `curl` registration block per template. +- [`samples/templates/solr/README.md`](../../samples/templates/solr/README.md) — + the two runnable Solr templates (`edismax_basic`, `boost_decay`). + +### Look up a specific parameter + +Each tunable knob has a per-engine reference page with native + unified +names, valid ranges, "when to tune", caveats, and the templates that +declare it. + +- [`docs/06_vendor_docs/elasticsearch-tunable-params.md`](../06_vendor_docs/elasticsearch-tunable-params.md) +- [`docs/06_vendor_docs/opensearch-tunable-params.md`](../06_vendor_docs/opensearch-tunable-params.md) + (covers OpenSearch's hybrid normalization-processor — NOT the ES + `rrf` retriever) +- [`docs/06_vendor_docs/solr-tunable-params.md`](../06_vendor_docs/solr-tunable-params.md) + (grounded in the checked-in Solr 9 / 10 ref-guide source) + +### The rest of the project - The full feature set is in [`docs/02_product/mvp1-user-stories.md`](../02_product/mvp1-user-stories.md). - The architectural decisions are in diff --git a/ui/src/__tests__/components/studies/create-study-modal.template-summary.test.tsx b/ui/src/__tests__/components/studies/create-study-modal.template-summary.test.tsx new file mode 100644 index 00000000..26b24c74 --- /dev/null +++ b/ui/src/__tests__/components/studies/create-study-modal.template-summary.test.tsx @@ -0,0 +1,195 @@ +// SPDX-FileCopyrightText: 2026 soundminds.ai +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * FR-7 modal-level wiring test for `chore_template_library_expansion` + * Story 3.1 — verifies the Step-3 template summary actually renders + * when the operator picks a template registered under a recommended + * name, and renders nothing (graceful miss) when the registered name + * is unrecognized. + * + * GPT-5.5 final-review cycle-2 finding on PR #416 — accepted: the + * library-level test at `ui/src/__tests__/lib/template-descriptions.test.ts` + * verifies the contract of `descriptionFor` / `cheatsheetUrlFor` but + * does not exercise the JSX wire-up in `create-study-modal.tsx`. A + * regression that rips out the summary `

` block would pass the + * lib-level test but break the operator UX. This test catches that. + */ + +import { http, HttpResponse } from 'msw'; +import { describe, expect, it, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { type ReactNode } from 'react'; + +import { TooltipProvider } from '@/components/ui/tooltip'; + +import { server } from '../../setup'; + +// Radix Select crashes inside jsdom's portal handling for this many-Select +// modal — replace with the shared native `