diff --git a/api/oss/src/apis/fastapi/evaluators/models.py b/api/oss/src/apis/fastapi/evaluators/models.py index 9bbf16b796..d3e15c97fa 100644 --- a/api/oss/src/apis/fastapi/evaluators/models.py +++ b/api/oss/src/apis/fastapi/evaluators/models.py @@ -1,7 +1,6 @@ from typing import Optional, List, Dict, Any -from pydantic import BaseModel - +from pydantic import BaseModel, Field from oss.src.utils.exceptions import Support from oss.src.core.shared.dtos import ( @@ -47,244 +46,588 @@ class EvaluatorCreateRequest(BaseModel): - evaluator: EvaluatorCreate + """Body for creating an evaluator artifact. + + Creating an evaluator also provisions its first variant and its initial + revision. The evaluator shares the artifact / variant / revision model + used across versioned resources — see the Versioning guide. + """ + + evaluator: EvaluatorCreate = Field( + description="Evaluator payload (slug, name, flags, data). Slug is required and scoped to the project.", + ) class EvaluatorEditRequest(BaseModel): - evaluator: EvaluatorEdit + """Body for editing the metadata of an existing evaluator artifact.""" + + evaluator: EvaluatorEdit = Field( + description="Evaluator edit payload. Requires the evaluator `id`. Renaming is temporarily disabled.", + ) class EvaluatorQueryRequest(BaseModel): - evaluator: Optional[EvaluatorQuery] = None + """Body for filtering evaluators. See the Query Pattern guide for field semantics.""" + + evaluator: Optional[EvaluatorQuery] = Field( + default=None, + description="Filter on evaluator attributes (flags, tags, meta).", + ) # - evaluator_refs: Optional[List[Reference]] = None + evaluator_refs: Optional[List[Reference]] = Field( + default=None, + description="Restrict the query to these evaluators. Accepts `id` or `slug` per reference.", + ) # - include_archived: Optional[bool] = None + include_archived: Optional[bool] = Field( + default=None, + description="When true, include soft-deleted evaluators in the response. Defaults to false.", + ) # - windowing: Optional[Windowing] = None + windowing: Optional[Windowing] = Field( + default=None, + description="Cursor-based pagination controls (limit, order, next, newest, oldest).", + ) class EvaluatorForkRequest(BaseModel): - evaluator: EvaluatorFork + """Body for forking an evaluator variant into a new variant. + + Forking copies the variant's history into a new branch so experiments + can proceed without touching the original. + """ + + evaluator: EvaluatorFork = Field( + description="Fork payload. References the source variant or revision and the target evaluator.", + ) class EvaluatorResponse(Support): - count: int = 0 - evaluator: Optional[Evaluator] = None + """Envelope for a single evaluator response.""" + + count: int = Field( + default=0, description="1 when an evaluator is returned, 0 otherwise." + ) + evaluator: Optional[Evaluator] = Field( + default=None, + description="The evaluator artifact, or null when none matched.", + ) class EvaluatorsResponse(Support): - count: int = 0 - evaluators: List[Evaluator] = [] + """Envelope for a list of evaluators.""" + + count: int = Field(default=0, description="Number of evaluators in `evaluators`.") + evaluators: List[Evaluator] = Field( + default_factory=list, + description="Matching evaluator artifacts.", + ) # EVALUATOR VARIANTS ----------------------------------------------------------- class EvaluatorVariantCreateRequest(BaseModel): - evaluator_variant: EvaluatorVariantCreate + """Body for creating a new variant on an existing evaluator.""" + + evaluator_variant: EvaluatorVariantCreate = Field( + description="Variant payload. Requires the parent `evaluator_id`.", + ) class EvaluatorVariantEditRequest(BaseModel): - evaluator_variant: EvaluatorVariantEdit + """Body for editing a variant's metadata.""" + + evaluator_variant: EvaluatorVariantEdit = Field( + description="Variant edit payload. Requires the variant `id`.", + ) class EvaluatorVariantQueryRequest(BaseModel): - evaluator_variant: Optional[EvaluatorVariantQuery] = None + """Body for filtering evaluator variants. Supports scoping to one or more evaluators.""" + + evaluator_variant: Optional[EvaluatorVariantQuery] = Field( + default=None, + description="Filter on variant attributes.", + ) # - evaluator_refs: Optional[List[Reference]] = None - evaluator_variant_refs: Optional[List[Reference]] = None + evaluator_refs: Optional[List[Reference]] = Field( + default=None, + description="Restrict to variants belonging to these evaluators.", + ) + evaluator_variant_refs: Optional[List[Reference]] = Field( + default=None, + description="Restrict to these specific variants.", + ) # - include_archived: Optional[bool] = None + include_archived: Optional[bool] = Field( + default=None, + description="When true, include soft-deleted variants. Defaults to false.", + ) # - windowing: Optional[Windowing] = None + windowing: Optional[Windowing] = Field( + default=None, + description="Cursor-based pagination controls.", + ) class EvaluatorVariantForkRequest(BaseModel): # TODO: FIX ME - source_evaluator_variant_ref: Reference - target_evaluator_ref: Reference - slug: Optional[str] = None - name: Optional[str] = None - description: Optional[str] = None + """Legacy fork payload. Use `EvaluatorForkRequest` for new code.""" + + source_evaluator_variant_ref: Reference = Field( + description="Variant to fork from.", + ) + target_evaluator_ref: Reference = Field( + description="Evaluator that will receive the new variant.", + ) + slug: Optional[str] = Field(default=None, description="Slug for the new variant.") + name: Optional[str] = Field( + default=None, description="Display name for the new variant." + ) + description: Optional[str] = Field( + default=None, description="Optional description." + ) class EvaluatorRevisionsLogRequest(BaseModel): - evaluator: EvaluatorRevisionsLog + """Body for listing the revision log of an evaluator variant.""" + + evaluator: EvaluatorRevisionsLog = Field( + description="Log request scoped to an evaluator / variant / revision by id, slug, or version.", + ) class EvaluatorVariantResponse(Support): - count: int = 0 - evaluator_variant: Optional[EvaluatorVariant] = None + """Envelope for a single evaluator variant.""" + + count: int = Field( + default=0, description="1 when a variant is returned, 0 otherwise." + ) + evaluator_variant: Optional[EvaluatorVariant] = Field( + default=None, + description="The evaluator variant, or null when none matched.", + ) class EvaluatorVariantsResponse(Support): - count: int = 0 - evaluator_variants: List[EvaluatorVariant] = [] + """Envelope for a list of evaluator variants.""" + + count: int = Field( + default=0, description="Number of variants in `evaluator_variants`." + ) + evaluator_variants: List[EvaluatorVariant] = Field( + default_factory=list, + description="Matching evaluator variants.", + ) # EVALUATOR REVISIONS ---------------------------------------------------------- class EvaluatorRevisionCreateRequest(BaseModel): - evaluator_revision: EvaluatorRevisionCreate + """Body for creating a new revision (commit) on an evaluator variant.""" + + evaluator_revision: EvaluatorRevisionCreate = Field( + description="Revision payload. Requires the parent `evaluator_variant_id` and a `data` object.", + ) class EvaluatorRevisionEditRequest(BaseModel): - evaluator_revision: EvaluatorRevisionEdit + """Body for editing a revision's mutable fields (currently limited; payload data is immutable).""" + + evaluator_revision: EvaluatorRevisionEdit = Field( + description="Revision edit payload. Requires the revision `id`.", + ) class EvaluatorRevisionQueryRequest(BaseModel): - evaluator_revision: Optional[EvaluatorRevisionQuery] = None + """Body for filtering evaluator revisions. Supports scoping to evaluators, variants, or specific revisions.""" + + evaluator_revision: Optional[EvaluatorRevisionQuery] = Field( + default=None, + description="Filter on revision attributes.", + ) # - evaluator_refs: Optional[List[Reference]] = None - evaluator_variant_refs: Optional[List[Reference]] = None - evaluator_revision_refs: Optional[List[Reference]] = None + evaluator_refs: Optional[List[Reference]] = Field( + default=None, + description="Restrict to revisions under these evaluators.", + ) + evaluator_variant_refs: Optional[List[Reference]] = Field( + default=None, + description="Restrict to revisions under these variants.", + ) + evaluator_revision_refs: Optional[List[Reference]] = Field( + default=None, + description="Restrict to these specific revisions.", + ) # - include_archived: Optional[bool] = None + include_archived: Optional[bool] = Field( + default=None, + description="When true, include soft-deleted revisions. Defaults to false.", + ) # - windowing: Optional[Windowing] = None - resolve: Optional[bool] = None # Optionally resolve embeds on query + windowing: Optional[Windowing] = Field( + default=None, + description="Cursor-based pagination controls.", + ) + resolve: Optional[bool] = Field( + default=None, + description="When true, resolve embedded references on each returned revision's `data`.", + ) class EvaluatorRevisionCommitRequest(BaseModel): - evaluator_revision_commit: EvaluatorRevisionCommit + """Body for committing a new revision on a variant.""" + + evaluator_revision_commit: EvaluatorRevisionCommit = Field( + description="Commit payload carrying the `evaluator_variant_id`, optional commit `message`, and the revision `data`.", + ) class EvaluatorRevisionRetrieveRequest(BaseModel): - evaluator_ref: Optional[Reference] = None - evaluator_variant_ref: Optional[Reference] = None - evaluator_revision_ref: Optional[Reference] = None - environment_ref: Optional[Reference] = None - environment_variant_ref: Optional[Reference] = None - environment_revision_ref: Optional[Reference] = None - key: Optional[str] = None - resolve: Optional[bool] = None # Optionally resolve embeds on retrieve + """Body for retrieving one revision, either by direct reference or through an environment key. + + Provide one of: an evaluator / variant / revision reference, or an environment reference plus `key`. + """ + + evaluator_ref: Optional[Reference] = Field( + default=None, + description="Retrieve the latest revision of this evaluator.", + ) + evaluator_variant_ref: Optional[Reference] = Field( + default=None, + description="Retrieve the latest revision on this variant.", + ) + evaluator_revision_ref: Optional[Reference] = Field( + default=None, + description="Retrieve this specific revision.", + ) + environment_ref: Optional[Reference] = Field( + default=None, + description="Environment to resolve through. Requires `key`.", + ) + environment_variant_ref: Optional[Reference] = Field( + default=None, + description="Environment variant to resolve through. Requires `key`.", + ) + environment_revision_ref: Optional[Reference] = Field( + default=None, + description="Specific environment revision to resolve through. Requires `key`.", + ) + key: Optional[str] = Field( + default=None, + description="Named deployment key inside the environment revision. Required with environment refs.", + ) + resolve: Optional[bool] = Field( + default=None, + description="When true, resolve embedded references on the returned revision's `data`.", + ) class EvaluatorRevisionDeployRequest(BaseModel): - evaluator_ref: Optional[Reference] = None - evaluator_variant_ref: Optional[Reference] = None - evaluator_revision_ref: Optional[Reference] = None - environment_ref: Optional[Reference] = None - environment_variant_ref: Optional[Reference] = None - environment_revision_ref: Optional[Reference] = None - key: Optional[str] = None - message: Optional[str] = None + """Body for pinning an evaluator revision into an environment revision under a key.""" + + evaluator_ref: Optional[Reference] = Field( + default=None, + description="Evaluator to deploy (latest revision).", + ) + evaluator_variant_ref: Optional[Reference] = Field( + default=None, + description="Variant to deploy (latest revision on this variant).", + ) + evaluator_revision_ref: Optional[Reference] = Field( + default=None, + description="Specific revision to deploy.", + ) + environment_ref: Optional[Reference] = Field( + default=None, + description="Target environment.", + ) + environment_variant_ref: Optional[Reference] = Field( + default=None, + description="Target environment variant.", + ) + environment_revision_ref: Optional[Reference] = Field( + default=None, + description="Target environment revision.", + ) + key: Optional[str] = Field( + default=None, + description="Named key under which the revision is pinned. Defaults to `.revision`.", + ) + message: Optional[str] = Field( + default=None, + description="Commit message stored on the environment revision that records the deployment.", + ) class EvaluatorRevisionResponse(Support): - count: int = 0 - evaluator_revision: Optional[EvaluatorRevision] = None - resolution_info: Optional[ResolutionInfo] = None # Included when resolve=True + """Envelope for a single evaluator revision.""" + + count: int = Field( + default=0, description="1 when a revision is returned, 0 otherwise." + ) + evaluator_revision: Optional[EvaluatorRevision] = Field( + default=None, + description="The evaluator revision, or null when none matched.", + ) + resolution_info: Optional[ResolutionInfo] = Field( + default=None, + description="Embed-resolution metadata. Populated when `resolve=true` was requested.", + ) class EvaluatorRevisionsResponse(Support): - count: int = 0 - evaluator_revisions: List[EvaluatorRevision] = [] + """Envelope for a list of evaluator revisions.""" + + count: int = Field( + default=0, description="Number of revisions in `evaluator_revisions`." + ) + evaluator_revisions: List[EvaluatorRevision] = Field( + default_factory=list, + description="Matching evaluator revisions.", + ) # SIMPLE EVALUATORS ------------------------------------------------------------ class SimpleEvaluatorCreateRequest(BaseModel): - evaluator: SimpleEvaluatorCreate + """Body for creating an evaluator via the simple surface. + + Collapses artifact, variant, and first revision into one call. The + response returns the same flat shape that `/simple/evaluators/query` + exposes. + """ + + evaluator: SimpleEvaluatorCreate = Field( + description="Simple evaluator payload (slug, name, flags, and `data` with `uri` + `parameters`).", + ) class SimpleEvaluatorEditRequest(BaseModel): - evaluator: SimpleEvaluatorEdit + """Body for editing an evaluator via the simple surface.""" + + evaluator: SimpleEvaluatorEdit = Field( + description="Simple evaluator edit payload. Requires the evaluator `id`. Renaming is temporarily disabled.", + ) class SimpleEvaluatorQueryRequest(BaseModel): - evaluator: Optional[SimpleEvaluatorQuery] = None + """Body for filtering evaluators via the simple surface.""" + + evaluator: Optional[SimpleEvaluatorQuery] = Field( + default=None, + description="Filter on evaluator attributes (slug, slugs, flags, meta).", + ) # - evaluator_refs: Optional[List[Reference]] = None + evaluator_refs: Optional[List[Reference]] = Field( + default=None, + description="Restrict to these evaluators.", + ) # - include_archived: Optional[bool] = False + include_archived: Optional[bool] = Field( + default=False, + description="When true, include soft-deleted evaluators.", + ) # - windowing: Optional[Windowing] = None + windowing: Optional[Windowing] = Field( + default=None, + description="Cursor-based pagination controls.", + ) class SimpleEvaluatorResponse(Support): - count: int = 0 - evaluator: Optional[SimpleEvaluator] = None + """Envelope for a single simple evaluator.""" + + count: int = Field( + default=0, description="1 when an evaluator is returned, 0 otherwise." + ) + evaluator: Optional[SimpleEvaluator] = Field( + default=None, + description="The flat evaluator record with latest variant and revision merged into `data`.", + ) class SimpleEvaluatorsResponse(Support): - count: int = 0 - evaluators: List[SimpleEvaluator] = [] + """Envelope for a list of simple evaluators.""" + + count: int = Field(default=0, description="Number of evaluators in `evaluators`.") + evaluators: List[SimpleEvaluator] = Field( + default_factory=list, + description="Matching flat evaluator records.", + ) # EVALUATOR REVISION RESOLUTION ------------------------------------------------ class EvaluatorRevisionResolveRequest(BaseModel): - evaluator_ref: Optional[Reference] = None - evaluator_variant_ref: Optional[Reference] = None - evaluator_revision_ref: Optional[Reference] = None + """Body for resolving embedded references on an evaluator revision's `data`.""" + + evaluator_ref: Optional[Reference] = Field( + default=None, + description="Resolve the latest revision of this evaluator.", + ) + evaluator_variant_ref: Optional[Reference] = Field( + default=None, + description="Resolve the latest revision on this variant.", + ) + evaluator_revision_ref: Optional[Reference] = Field( + default=None, + description="Resolve this specific revision.", + ) # - max_depth: Optional[int] = 10 - max_embeds: Optional[int] = 100 - error_policy: Optional[ErrorPolicy] = ErrorPolicy.EXCEPTION + max_depth: Optional[int] = Field( + default=10, + description="Maximum recursion depth when following embedded references. Defaults to 10.", + ) + max_embeds: Optional[int] = Field( + default=100, + description="Maximum number of embeds to resolve. Defaults to 100.", + ) + error_policy: Optional[ErrorPolicy] = Field( + default=ErrorPolicy.EXCEPTION, + description="How to handle embed-resolution errors (`exception` or `fallback`).", + ) class EvaluatorRevisionResolveResponse(Support): - count: int = 0 - evaluator_revision: Optional[EvaluatorRevision] = None - resolution_info: Optional[ResolutionInfo] = None + """Envelope for a resolved evaluator revision.""" + + count: int = Field( + default=0, description="1 when a revision was resolved, 0 otherwise." + ) + evaluator_revision: Optional[EvaluatorRevision] = Field( + default=None, + description="The resolved revision.", + ) + resolution_info: Optional[ResolutionInfo] = Field( + default=None, + description="Diagnostic information about the resolution pass (depth, embed count, errors).", + ) # EVALUATOR TEMPLATES ---------------------------------------------------------- class EvaluatorTemplate(BaseModel): - """Static evaluator template definition (built-in evaluator types).""" - - name: str - key: str - direct_use: bool - settings_presets: Optional[List[Dict[str, Any]]] = None - settings_template: Dict[str, Any] - outputs_schema: Optional[Dict[str, Any]] = None - description: Optional[str] = None - oss: Optional[bool] = False - requires_llm_api_keys: Optional[bool] = False - tags: List[str] = [] - archived: Optional[bool] = False + """Static evaluator template definition (built-in evaluator types). + + Templates are shipped with the product and describe the available + evaluator types. They are read-only and separate from user-owned + evaluator artifacts. + """ + + name: str = Field(description="Human-readable template name.") + key: str = Field( + description="Stable template identifier, used to create evaluators from the template." + ) + direct_use: bool = Field( + description="Whether the template can be used without further configuration." + ) + settings_presets: Optional[List[Dict[str, Any]]] = Field( + default=None, + description="Preset parameter configurations shipped with the template.", + ) + settings_template: Dict[str, Any] = Field( + description="JSON Schema describing the template's configurable parameters.", + ) + outputs_schema: Optional[Dict[str, Any]] = Field( + default=None, + description="JSON Schema describing the template's evaluator output shape.", + ) + description: Optional[str] = Field( + default=None, description="Template description." + ) + oss: Optional[bool] = Field( + default=False, description="True when the template is available in OSS builds." + ) + requires_llm_api_keys: Optional[bool] = Field( + default=False, + description="True when the template calls an LLM provider and requires an API key.", + ) + tags: List[str] = Field( + default_factory=list, description="Tags for grouping templates." + ) + archived: Optional[bool] = Field( + default=False, + description="True when the template is deprecated. Hidden unless `include_archived=true`.", + ) class EvaluatorTemplatesResponse(Support): - count: int = 0 - templates: List[EvaluatorTemplate] = [] + """Envelope for a list of evaluator templates.""" + + count: int = Field(default=0, description="Number of templates in `templates`.") + templates: List[EvaluatorTemplate] = Field( + default_factory=list, + description="Built-in evaluator templates.", + ) # EVALUATORS CATALOG ----------------------------------------------------------- class EvaluatorCatalogTypeResponse(Support): - count: int = 0 - type: Optional[EvaluatorCatalogType] = None + """Envelope for a single catalog type.""" + + count: int = Field(default=0, description="1 when a type is returned, 0 otherwise.") + type: Optional[EvaluatorCatalogType] = Field( + default=None, + description="The catalog type, or null when none matched.", + ) class EvaluatorCatalogTypesResponse(Support): - count: int = 0 - types: List[EvaluatorCatalogType] = [] + """Envelope for a list of catalog types.""" + + count: int = Field(default=0, description="Number of types in `types`.") + types: List[EvaluatorCatalogType] = Field( + default_factory=list, + description="JSON schema types the evaluator catalog understands.", + ) class EvaluatorCatalogTemplateResponse(Support): - count: int = 0 - template: Optional[EvaluatorCatalogTemplate] = None + """Envelope for a single catalog template.""" + + count: int = Field( + default=0, description="1 when a template is returned, 0 otherwise." + ) + template: Optional[EvaluatorCatalogTemplate] = Field( + default=None, + description="The catalog template, or null when none matched.", + ) class EvaluatorCatalogTemplatesResponse(Support): - count: int = 0 - templates: List[EvaluatorCatalogTemplate] = [] + """Envelope for a list of catalog templates.""" + + count: int = Field(default=0, description="Number of templates in `templates`.") + templates: List[EvaluatorCatalogTemplate] = Field( + default_factory=list, + description="Evaluator catalog templates (blueprints for creating evaluators).", + ) class EvaluatorCatalogPresetResponse(Support): - count: int = 0 - preset: Optional[EvaluatorCatalogPreset] = None + """Envelope for a single catalog preset.""" + + count: int = Field( + default=0, description="1 when a preset is returned, 0 otherwise." + ) + preset: Optional[EvaluatorCatalogPreset] = Field( + default=None, + description="The catalog preset, or null when none matched.", + ) class EvaluatorCatalogPresetsResponse(Support): - count: int = 0 - presets: List[EvaluatorCatalogPreset] = [] + """Envelope for a list of catalog presets.""" + + count: int = Field(default=0, description="Number of presets in `presets`.") + presets: List[EvaluatorCatalogPreset] = Field( + default_factory=list, + description="Named parameter presets defined against a template.", + ) diff --git a/api/oss/src/apis/fastapi/evaluators/router.py b/api/oss/src/apis/fastapi/evaluators/router.py index c72bde981b..88c5518952 100644 --- a/api/oss/src/apis/fastapi/evaluators/router.py +++ b/api/oss/src/apis/fastapi/evaluators/router.py @@ -437,6 +437,13 @@ def __init__( async def list_evaluator_catalog_types( self, ) -> EvaluatorCatalogTypesResponse: + """List the JSON schema types the evaluator catalog understands. + + Types are static metadata shipped with the product. Use this when + rendering a catalog UI or validating that a template's schema is + supported. See the Evaluators guide for how the catalog relates + to user-owned evaluator artifacts. + """ types = [ EvaluatorCatalogType(**type_data.model_dump()) for type_data in get_workflow_catalog_types() @@ -456,6 +463,14 @@ async def list_evaluator_catalog_templates( *, include_archived: Optional[bool] = None, ) -> EvaluatorCatalogTemplatesResponse: + """List evaluator templates from the catalog. + + Templates are blueprints that describe an evaluator's handler + URI, JSON schemas, and default configuration. Pass + `include_archived=true` to include deprecated templates. Use the + returned `key` with `/catalog/templates/{template_key}/presets/` + to list its presets. + """ templates = [ _registry_entry_to_catalog_template(entry) for entry in get_filtered_workflow_catalog_templates(is_evaluator=True) @@ -476,6 +491,12 @@ async def fetch_evaluator_catalog_template( *, template_key: str, ) -> EvaluatorCatalogTemplateResponse: + """Fetch one evaluator template by key. + + Returns an empty envelope (`count: 0`) when no template matches + the key. Template keys come from + `GET /catalog/templates/`. + """ entry = get_workflow_catalog_template( template_key=template_key, is_evaluator=True, @@ -497,6 +518,12 @@ async def list_evaluator_catalog_presets( template_key: str, include_archived: Optional[bool] = None, ) -> EvaluatorCatalogPresetsResponse: + """List presets defined against one evaluator template. + + A preset is a named set of parameter values pre-filled against + the template. Use the returned `key` to fetch a specific preset + via `GET /catalog/templates/{template_key}/presets/{preset_key}`. + """ presets = [ EvaluatorCatalogPreset(**preset.model_dump()) for preset in get_filtered_workflow_catalog_presets( @@ -521,6 +548,12 @@ async def fetch_evaluator_catalog_preset( template_key: str, preset_key: str, ) -> EvaluatorCatalogPresetResponse: + """Fetch one evaluator preset by template and preset key. + + Presets are not separate entities; they are metadata. Use the + returned preset payload as the starting point when creating a + new evaluator from a template. See the Evaluators guide. + """ preset_data = get_workflow_catalog_preset( template_key=template_key, preset_key=preset_key, @@ -546,6 +579,14 @@ async def create_evaluator( # evaluator_create_request: EvaluatorCreateRequest, ) -> EvaluatorResponse: + """Create an evaluator artifact, its first variant, and its initial revision. + + Use this endpoint when you already know you want to manage the + artifact / variant / revision layers independently. For a + one-shot "create and forget" call that returns a flat record, + see `POST /simple/evaluators/`. See the Versioning guide for + commit semantics. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -576,6 +617,12 @@ async def fetch_evaluator( *, evaluator_id: UUID, ) -> EvaluatorResponse: + """Fetch an evaluator artifact by id. + + Returns the artifact-level record (slug, name, flags, lifecycle) + without variant or revision data. Use the variant and revision + endpoints to retrieve those layers. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -604,6 +651,13 @@ async def edit_evaluator( # evaluator_edit_request: EvaluatorEditRequest, ) -> EvaluatorResponse: + """Edit an evaluator artifact's metadata. + + Edits are limited to metadata fields (description, tags, meta). + Renaming is temporarily disabled and returns 400. To change + evaluator behavior, commit a new revision on the variant — see + `/evaluators/revisions/commit`. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -655,6 +709,12 @@ async def archive_evaluator( *, evaluator_id: UUID, ) -> EvaluatorResponse: + """Soft-delete an evaluator artifact. + + Sets `deleted_at` on the evaluator and hides it from subsequent + `/query` responses unless `include_archived=true`. Revision IDs + remain resolvable so historical traces stay intact. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -682,6 +742,11 @@ async def unarchive_evaluator( *, evaluator_id: UUID, ) -> EvaluatorResponse: + """Restore a soft-deleted evaluator artifact. + + Clears `deleted_at` on the evaluator so it re-appears in `/query` + responses. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -710,6 +775,13 @@ async def query_evaluators( *, evaluator_query_request: EvaluatorQueryRequest, ) -> EvaluatorsResponse: + """Query evaluator artifacts with filters and pagination. + + Returns artifact-level records only. The request body follows + the shared query pattern (filter + refs + windowing). Send `{}` + to list all evaluators in the project. See the Query Pattern + guide. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -746,6 +818,12 @@ async def create_evaluator_variant( *, evaluator_variant_create_request: EvaluatorVariantCreateRequest, ) -> EvaluatorVariantResponse: + """Create a new variant on an existing evaluator. + + A variant is a named branch of the evaluator's history. New + revisions committed to this variant do not touch other variants. + See the Versioning guide. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -775,6 +853,12 @@ async def fetch_evaluator_variant( *, evaluator_variant_id: UUID, ) -> EvaluatorVariantResponse: + """Fetch an evaluator variant by id. + + Returns the variant record (slug, flags, lifecycle) without the + committed revisions. Use `/evaluators/revisions/retrieve` to + read the variant's current revision payload. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -805,6 +889,12 @@ async def edit_evaluator_variant( # evaluator_variant_edit_request: EvaluatorVariantEditRequest, ) -> EvaluatorVariantResponse: + """Edit a variant's metadata. + + Edits only touch variant-level metadata. To change evaluator + behavior commit a new revision via + `/evaluators/revisions/commit`. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -839,6 +929,12 @@ async def archive_evaluator_variant( *, evaluator_variant_id: UUID, ) -> EvaluatorVariantResponse: + """Soft-delete an evaluator variant. + + Sets `deleted_at` on the variant. Its revisions stay resolvable + by id; they are hidden from `/query` unless the caller sets + `include_archived=true`. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -868,6 +964,7 @@ async def unarchive_evaluator_variant( *, evaluator_variant_id: UUID, ) -> EvaluatorVariantResponse: + """Restore a soft-deleted evaluator variant.""" if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -899,6 +996,12 @@ async def query_evaluator_variants( parse_evaluator_variant_query_request_from_params ), ) -> EvaluatorVariantsResponse: + """Query evaluator variants with filters, reference scoping, and pagination. + + Accepts parameters from both the query string and a JSON body; + the two are merged. Use `evaluator_refs` to scope to one or more + evaluators, or `evaluator_variant_refs` for specific variants. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -955,6 +1058,13 @@ async def fork_evaluator_variant( # evaluator_variant_fork_request: EvaluatorForkRequest, ): + """Fork an evaluator variant into a new variant. + + Creates a new branch whose initial revision is copied from the + source. Use this to experiment without touching the original. + The returned variant has a fresh id and slug but inherits + lineage metadata from its source. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1004,6 +1114,15 @@ async def deploy_evaluator_revision( *, evaluator_deploy_request: EvaluatorRevisionDeployRequest, ) -> EvaluatorRevisionResponse: + """Pin an evaluator revision into an environment revision under a key. + + Requires an evaluator ref (`evaluator_ref`, + `evaluator_variant_ref`, or `evaluator_revision_ref`) and an + environment ref. When `key` is omitted it defaults to + `.revision`. The deployment is recorded as a + new commit on the environment revision. See the Evaluators + guide for the deployment model. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1156,6 +1275,16 @@ async def retrieve_evaluator_revision( *, evaluator_revision_retrieve_request: EvaluatorRevisionRetrieveRequest, ) -> EvaluatorRevisionResponse: + """Retrieve one evaluator revision, either directly or via an environment key. + + Provide one of: + an evaluator / variant / revision reference (returns that + revision, or the latest revision on the variant or evaluator), + or an environment reference plus `key` (returns the revision + currently pinned to that key). Supplying both forms returns 400. + Pass `resolve=true` to expand embedded references on the + returned payload. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1265,6 +1394,12 @@ async def create_evaluator_revision( *, evaluator_revision_create_request: EvaluatorRevisionCreateRequest, ) -> EvaluatorRevisionResponse: + """Create a new revision on an evaluator variant. + + Prefer `/evaluators/revisions/commit` for the standard commit + flow. This endpoint exists for internal create paths that need + to insert a revision without the commit semantics. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1293,6 +1428,13 @@ async def fetch_evaluator_revision( *, evaluator_revision_id: UUID, ) -> EvaluatorRevisionResponse: + """Fetch a specific evaluator revision by id. + + Returns the full revision including `data` (handler uri, + schemas, and parameters). To pick the latest revision on a + variant without knowing its id, use + `/evaluators/revisions/retrieve`. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1321,6 +1463,12 @@ async def edit_evaluator_revision( # evaluator_revision_edit_request: EvaluatorRevisionEditRequest, ) -> EvaluatorRevisionResponse: + """Edit a revision's metadata. + + Revision `data` is immutable once committed. This endpoint is + for metadata fields only (description, tags, meta). To change + evaluator behavior, commit a new revision instead. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1353,6 +1501,11 @@ async def archive_evaluator_revision( *, evaluator_revision_id: UUID, ) -> EvaluatorRevisionResponse: + """Soft-delete an evaluator revision. + + Archived revisions remain resolvable by id but are excluded from + revision logs and queries unless `include_archived=true`. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1380,6 +1533,7 @@ async def unarchive_evaluator_revision( *, evaluator_revision_id: UUID, ) -> EvaluatorRevisionResponse: + """Restore a soft-deleted evaluator revision.""" if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1408,6 +1562,13 @@ async def query_evaluator_revisions( *, evaluator_revision_query_request: EvaluatorRevisionQueryRequest, ) -> EvaluatorRevisionsResponse: + """Query evaluator revisions with filters, reference scoping, and pagination. + + Returns revision payloads. Use `evaluator_refs`, + `evaluator_variant_refs`, or `evaluator_revision_refs` to scope + the query. Pass `resolve=true` to expand embedded references on + each revision's `data`. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1459,6 +1620,12 @@ async def commit_evaluator_revision( *, evaluator_revision_commit_request: EvaluatorRevisionCommitRequest, ) -> EvaluatorRevisionResponse: + """Commit a new revision on an evaluator variant. + + The commit body carries the target `evaluator_variant_id`, an + optional `message`, and the revision `data` (handler uri, + schemas, parameters). A committed revision is immutable. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1486,6 +1653,12 @@ async def log_evaluator_revisions( *, evaluator_revisions_log_request: EvaluatorRevisionsLogRequest, ): + """List the revision log of an evaluator variant. + + Returns revisions in commit order. Scope the log by supplying + an evaluator, variant, or revision reference. Use the retrieve + endpoint to fetch a specific revision's full payload. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1514,6 +1687,13 @@ async def resolve_evaluator_revision( *, evaluator_revision_resolve_request: EvaluatorRevisionResolveRequest, ) -> EvaluatorRevisionResolveResponse: + """Resolve embedded references on an evaluator revision's `data`. + + Walks embedded references (for example, references to other + revisions or to secrets) up to `max_depth` and `max_embeds`. + The response includes a `resolution_info` block with counts, + depth reached, and errors according to `error_policy`. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1646,6 +1826,13 @@ async def create_simple_evaluator( # simple_evaluator_create_request: SimpleEvaluatorCreateRequest, ) -> SimpleEvaluatorResponse: + """Create an evaluator via the simple surface. + + Creates the artifact, its first variant, and its initial + revision in one call. Returns the flat evaluator record + (latest revision merged into `data`). Use this when you do not + need to manage variants or revisions directly. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1678,6 +1865,11 @@ async def fetch_simple_evaluator( *, evaluator_id: UUID, ) -> SimpleEvaluatorResponse: + """Fetch one evaluator via the simple surface. + + Returns the flat evaluator record including its current variant + and revision ids and the merged `data` payload. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1708,6 +1900,12 @@ async def edit_simple_evaluator( # simple_evaluator_edit_request: SimpleEvaluatorEditRequest, ) -> SimpleEvaluatorResponse: + """Edit an evaluator via the simple surface. + + Touches metadata and (when `data` is supplied) commits a new + revision on the evaluator's variant. Renaming is temporarily + disabled and returns 400. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1761,6 +1959,11 @@ async def archive_simple_evaluator( # TODO: FIX ME *, evaluator_id: UUID, ) -> SimpleEvaluatorResponse: + """Soft-delete an evaluator via the simple surface. + + Archives the underlying artifact. Historical traces that + reference specific revision ids remain resolvable. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1790,6 +1993,7 @@ async def unarchive_simple_evaluator( # TODO: FIX ME request: Request, evaluator_id: UUID, ) -> SimpleEvaluatorResponse: + """Restore a soft-deleted evaluator via the simple surface.""" if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1818,6 +2022,12 @@ async def list_simple_evaluators( self, request: Request, ) -> SimpleEvaluatorsResponse: + """List evaluators via the simple surface. + + Convenience wrapper that calls `query_simple_evaluators` with + the default evaluator flag filter. Prefer the query endpoint for + pagination and filtering. + """ simple_evaluator_query_request = SimpleEvaluatorQueryRequest( evaluator=SimpleEvaluatorQuery( flags=SimpleEvaluatorQueryFlags( @@ -1840,6 +2050,13 @@ async def query_simple_evaluators( # TODO: FIX ME *, simple_evaluator_query_request: SimpleEvaluatorQueryRequest, ) -> SimpleEvaluatorsResponse: + """Query evaluators via the simple surface with filters and pagination. + + Returns flat evaluator records (one per artifact with its + latest variant and revision merged into `data`). Send `{}` to + list all evaluators in the project. See the Query Pattern + guide. + """ if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1875,10 +2092,12 @@ async def list_evaluator_templates( *, include_archived: bool = False, ) -> EvaluatorTemplatesResponse: - """ - Returns the list of built-in evaluator templates. + """List the legacy built-in evaluator templates. - These are static evaluator type definitions (not user-created configs). + Returns static evaluator-type definitions shipped with the + product. Prefer the `/evaluators/catalog/*` endpoints for new + integrations; this endpoint is kept for older clients. Pass + `include_archived=true` to include deprecated templates. """ from oss.src.resources.evaluators.evaluators import get_all_evaluators diff --git a/docs/docs/reference/api-guide/07-evaluators.mdx b/docs/docs/reference/api-guide/07-evaluators.mdx new file mode 100644 index 0000000000..3f375dfdea --- /dev/null +++ b/docs/docs/reference/api-guide/07-evaluators.mdx @@ -0,0 +1,126 @@ +--- +title: "Evaluators" +description: "What an evaluator is, how it runs, and how the catalog and presets work." +sidebar_position: 7 +--- + +An **evaluator** is a runnable workflow that scores the outputs of other workflows or traces. Evaluators are versioned: each commit produces an immutable revision, and an evaluation pins the revision it ran so later commits do not retroactively change past results. + +Evaluators are a kind of workflow. An evaluator artifact shares its UUID with a workflow artifact (the same value serves as both `evaluator_id` and `workflow_id`). The workflow aliases let the generic workflow endpoints operate on evaluators when that's convenient. + +An evaluator can run in four places: + +- In the playground, against any sample input you provide. +- In an offline evaluation, against a testset. +- In online evaluation, against incoming traces. +- Called directly by an API client. + +Each run produces an **annotation-type trace**. The annotation links to the trace being evaluated through `ag.references.*` and records the evaluator's inputs, parameters, and outputs. + +For commit semantics, `include_archived`, and how revision IDs stay stable, see [Versioning](/reference/api-guide/versioning). The rest of this page covers what is specific to evaluators. + +## Body + +An evaluator's `data` payload is what each revision commits. It has three fields. + +| Field | What it describes | +|-------|------------------| +| `data.uri` | Locates the handler that contains the scoring logic. Can be a built-in identifier (for example, `agenta:builtin:auto_exact_match:v0`) or an HTTPS URL for a custom handler. | +| `data.schemas` | JSON Schemas for `parameters` (configuration), `inputs` (what the evaluator reads), and `outputs` (what it returns). | +| `data.parameters` | Configured values for the parameters (the shape is described in `data.schemas.parameters`). | + +## How it runs + +To invoke an evaluator, the system POSTs an invocation payload to the handler at `data.uri`. The payload carries the `inputs` (sourced from a testcase or trace), the pinned `parameters`, and any `settings` and `credentials` the handler needs. + +The handler returns a dictionary of feedback. Values can be a numeric score, a boolean, a string, or an array. The built-in `auto_exact_match` evaluator, for example, returns `{"success": true}` or `{"success": false}`. A custom LLM-as-judge can return multiple fields. + +The system writes the result to the annotation trace described above. The trace records what the evaluator saw, what it returned, and a reference to the evaluator revision that produced the score. + +## Catalog and presets + +The catalog ships the list of built-in evaluators. Each one is described as a **template** with these fields: + +- `key`: the template's stable identifier. +- `uri`: the built-in handler URI. +- `schemas`: JSON Schemas for `parameters`, `inputs`, and `outputs`. +- `presets`: named sets of pinned parameter values for that template (for example, the "Quality Rating" preset for the `feedback` template). + +A preset is not a separate entity. Creating an evaluator "from a preset" produces a new artifact, variant, and first revision whose `data` comes from the template URI plus the preset's pinned parameter values. After that the evaluator is an ordinary instance you can fork, commit, and archive. + +| Endpoint | Purpose | +|----------|---------| +| `GET /evaluators/catalog/types/` | List the JSON Schema types the catalog understands. | +| `GET /evaluators/catalog/templates/` | List evaluator templates. | +| `GET /evaluators/catalog/templates/{key}` | Fetch one template by key. | +| `GET /evaluators/catalog/templates/{key}/presets/` | List presets defined for a template. | +| `GET /evaluators/catalog/templates/{key}/presets/{preset_key}` | Fetch one preset. | + +## Simple endpoints + +The `/simple/evaluators/` surface collapses the artifact, variant, and latest revision into one flat record: + +```bash +curl "$AGENTA_HOST/api/simple/evaluators/019d952f-0000-0000-0000-000000000000" \ + -H "Authorization: ApiKey $AGENTA_API_KEY" +``` + +Use it when you want the "current evaluator" without tracking lineage. For commits, forks, or specific-revision retrieval, use `/evaluators/`, `/evaluators/variants/`, and `/evaluators/revisions/`. See [Simple Endpoints](/reference/api-guide/simple-endpoints) for the general pattern. + +## Relationship to evaluations + +[Evaluations](/reference/api-guide/evaluations) are configured with one or more evaluators that run against a testset. Each evaluation pins specific `evaluator_revision_id`s when the run is configured. Committing a new revision on the variant does not retroactively change a pinned run. + +## Example + +Create an evaluator that uses the built-in exact-match handler, commit a new revision, and retrieve it. + +```bash +# 1. Create the evaluator (and its first variant + revision). +curl -X POST "$AGENTA_HOST/api/simple/evaluators/" \ + -H "Content-Type: application/json" \ + -H "Authorization: ApiKey $AGENTA_API_KEY" \ + -d '{ + "evaluator": { + "slug": "exact-match-evaluator", + "name": "Exact Match Evaluator", + "data": { + "uri": "agenta:builtin:auto_exact_match:v0", + "parameters": { "correct_answer_key": "correct_answer" } + } + } + }' +``` + +The response returns the evaluator with its merged `data` (handler URL, JSON schemas, parameters) and its `id`, `variant_id`, and `revision_id`. + +```bash +# 2. Commit a new revision on the evaluator's variant. +curl -X POST "$AGENTA_HOST/api/evaluators/revisions/commit" \ + -H "Content-Type: application/json" \ + -H "Authorization: ApiKey $AGENTA_API_KEY" \ + -d '{ + "evaluator_revision_commit": { + "evaluator_variant_id": "019d952f-0000-0000-0000-000000000000", + "message": "Match on answer_text instead", + "data": { + "uri": "agenta:builtin:auto_exact_match:v0", + "parameters": { "correct_answer_key": "answer_text" } + } + } + }' +``` + +```bash +# 3. Retrieve the latest revision for the variant. +curl -X POST "$AGENTA_HOST/api/evaluators/revisions/retrieve" \ + -H "Content-Type: application/json" \ + -H "Authorization: ApiKey $AGENTA_API_KEY" \ + -d '{ + "evaluator_variant_ref": { "id": "019d952f-0000-0000-0000-000000000000" } + }' +``` + +## Lifecycle + +Evaluators, variants, and revisions are soft-deleted. Use `POST /evaluators/{id}/archive` and `POST /evaluators/{id}/unarchive` to flip `deleted_at`. The same pattern applies at the variant and revision level. Archiving an evaluator hides it from `/query` responses unless the request sets `include_archived: true`. See [Versioning](/reference/api-guide/versioning#archive-and-unarchive).