LCORE-1723: Cross-Encoder Reranking for enhanced RAG

docker-compose.yaml

@@ -24,6 +24,7 @@ services:
     environment:
       - BRAVE_SEARCH_API_KEY=${BRAVE_SEARCH_API_KEY:-}
       - TAVILY_SEARCH_API_KEY=${TAVILY_SEARCH_API_KEY:-}
+      - EXTERNAL_PROVIDERS_DIR=${EXTERNAL_PROVIDERS_DIR:-/opt/app-root/external_providers}
       # OpenAI
       - OPENAI_API_KEY=${OPENAI_API_KEY}
       - E2E_OPENAI_MODEL=${E2E_OPENAI_MODEL:-gpt-4o-mini}

docs/openapi.json

@@ -12023,6 +12023,11 @@
                         "$ref": "#/components/schemas/OkpConfiguration",
                         "title": "OKP configuration",
                         "description": "OKP provider settings. Only used when 'okp' is listed in rag.inline or rag.tool."
+                    },
+                    "reranker": {
+                        "$ref": "#/components/schemas/RerankerConfiguration",
+                        "title": "Reranker configuration",
+                        "description": "Configuration for neural reranking of RAG chunks using cross-encoder."
                     }
                 },
                 "additionalProperties": false,
@@ -17814,11 +17819,43 @@
                         ],
                         "title": "Source",
                         "description": "Index name identifying the knowledge source from configuration"
+                    },
+                    "document_id": {
+                        "anyOf": [
+                            {
+                                "type": "string"
+                            },
+                            {
+                                "type": "null"
+                            }
+                        ],
+                        "title": "Document Id",
+                        "description": "Document ID for preserving identity during deduplication"
                     }
                 },
                 "type": "object",
                 "title": "ReferencedDocument",
-                "description": "Model representing a document referenced in generating a response.\n\nAttributes:\n    doc_url: Url to the referenced doc.\n    doc_title: Title of the referenced doc."
+                "description": "Model representing a document referenced in generating a response.\n\nAttributes:\n    doc_url: Url to the referenced doc.\n    doc_title: Title of the referenced doc.\n    document_id: Document ID for preserving identity during deduplication."
+            },
+            "RerankerConfiguration": {
+                "properties": {
+                    "enabled": {
+                        "type": "boolean",
+                        "title": "Reranker enabled",
+                        "description": "When True, reranking applied to RAG chunks. When False, reranking is disabled and original scoring used.",
+                        "default": false
+                    },
+                    "model": {
+                        "type": "string",
+                        "title": "Reranker model",
+                        "description": "Cross-encoder model name for reranking RAG chunks. Defaults to 'cross-encoder/ms-marco-MiniLM-L6-v2' from sentence-transformers.",
+                        "default": "cross-encoder/ms-marco-MiniLM-L6-v2"
+                    }
+                },
+                "additionalProperties": false,
+                "type": "object",
+                "title": "RerankerConfiguration",
+                "description": "Reranker configuration for RAG chunk reranking."
             },
             "ResponseInput": {
                 "anyOf": [

pyproject.toml

@@ -72,6 +72,7 @@ dependencies = [
     "jinja2>=3.1.0",
     # To be able to fix multiple CVEs, also LCORE-1117
     "requests>=2.33.0",
+    # Used for RAG chunk reranking (cross-encoder)
     "datasets>=4.7.0",
     # Used for error tracking and monitoring
     "sentry-sdk[fastapi]>=2.58.0",

src/configuration.py

@@ -27,6 +27,7 @@
     OkpConfiguration,
     QuotaHandlersConfiguration,
     RagConfiguration,
+    RerankerConfiguration,
     RlsapiV1Configuration,
     ServiceConfiguration,
     SplunkConfiguration,
@@ -465,6 +466,13 @@ def okp(self) -> "OkpConfiguration":
             raise LogicError("logic error: configuration is not loaded")
         return self._configuration.okp
 
+    @property
+    def reranker(self) -> "RerankerConfiguration":
+        """Return reranker configuration."""
+        if self._configuration is None:
+            raise LogicError("logic error: configuration is not loaded")
+        return self._configuration.reranker
+
     @property
     def rag_id_mapping(self) -> dict[str, str]:
         """Return mapping from vector_db_id to rag_id from BYOK and OKP RAG config.

src/constants.py

@@ -181,6 +181,9 @@
 # Default embedding vector dimension for the sentence transformer model
 DEFAULT_EMBEDDING_DIMENSION: Final[int] = 768
 
+# Default sentence transformer cross encoder model for reranking RAG chunk scores
+DEFAULT_CROSS_ENCODER_MODEL: Final[str] = "cross-encoder/ms-marco-MiniLM-L6-v2"
+
 # quota limiters constants
 USER_QUOTA_LIMITER: Final[str] = "user_limiter"
 CLUSTER_QUOTA_LIMITER: Final[str] = "cluster_limiter"
@@ -192,6 +195,8 @@
 # Inline RAG constants
 BYOK_RAG_MAX_CHUNKS: Final[int] = 10  # retrieved from BYOK RAG
 OKP_RAG_MAX_CHUNKS: Final[int] = 5  # retrieved from OKP RAG
+# Score multiplier applied to BYOK chunks after cross-encoder reranking (Solr chunks unchanged)
+BYOK_RAG_RERANK_BOOST: Final[float] = 1.2
 
 # Solr OKP constants
 SOLR_VECTOR_SEARCH_DEFAULT_K: Final[int] = 5

src/models/common/turn_summary.py

@@ -31,6 +31,7 @@ class ReferencedDocument(BaseModel):
     Attributes:
         doc_url: Url to the referenced doc.
         doc_title: Title of the referenced doc.
+        document_id: Document ID for preserving identity during deduplication.
     """
 
     doc_url: Optional[AnyUrl] = Field(
@@ -46,6 +47,11 @@ class ReferencedDocument(BaseModel):
         description="Index name identifying the knowledge source from configuration",
     )
 
+    document_id: Optional[str] = Field(
+        default=None,
+        description="Document ID for preserving identity during deduplication",
+    )
+
 
 class RAGContext(BaseModel):
     """Result of building RAG context from all enabled pre-query RAG sources.

src/models/config.py

@@ -1817,6 +1817,34 @@ class OkpConfiguration(ConfigurationBase):
     )
 
 
+class RerankerConfiguration(ConfigurationBase):
+    """Reranker configuration for RAG chunk reranking."""
+
+    enabled: bool = Field(
+        default=False,
+        title="Reranker enabled",
+        description="When True, reranking applied to RAG chunks. "
+        "When False, reranking is disabled and original scoring used.",
+    )
+    model: str = Field(
+        default="cross-encoder/ms-marco-MiniLM-L6-v2",
+        title="Reranker model",
+        description="Cross-encoder model name for reranking RAG chunks. "
+        "Defaults to 'cross-encoder/ms-marco-MiniLM-L6-v2' from sentence-transformers.",
+    )
+
+    # Private attribute to track if this was explicitly configured
+    _explicitly_configured: bool = PrivateAttr(default=False)
+
+    @model_validator(mode="after")
+    def mark_as_explicitly_configured(self) -> Self:
+        """Mark this configuration as explicitly set when instantiated from user input."""
+        if self.model_fields_set:
+            self._explicitly_configured = True
+
+        return self
+
+
 class AzureEntraIdConfiguration(ConfigurationBase):
     """Microsoft Entra ID authentication attributes for Azure."""
 
@@ -1976,6 +2004,12 @@ class Configuration(ConfigurationBase):
         "in rag.inline or rag.tool.",
     )
 
+    reranker: RerankerConfiguration = Field(
+        default_factory=RerankerConfiguration,
+        title="Reranker configuration",
+        description="Configuration for neural reranking of RAG chunks using cross-encoder.",
+    )
+
     @model_validator(mode="after")
     def validate_mcp_auth_headers(self) -> Self:
         """
@@ -2078,6 +2112,43 @@ def validate_rlsapi_v1_quota_configuration(self) -> Self:
 
         return self
 
+    @model_validator(mode="after")
+    def validate_reranker_auto_enable(self) -> Self:
+        """Automatically enable reranker when both BYOK and OKP RAG are configured.
+
+        When users have both BYOK entries in byok_rag and OKP
+        configured in the RAG strategies, automatically
+        enable the reranker if it's not explicitly disabled. This improves result
+        quality when multiple knowledge sources are available.
+
+        Returns:
+            Self: The validated configuration instance with reranker potentially enabled.
+        """
+        # Check if BYOK RAG entries are configured
+        has_byok = len(self.byok_rag) > 0
+
+        # Check if OKP is configured in either inline or tool RAG strategies
+        # pylint: disable=no-member
+        has_okp = constants.OKP_RAG_ID in self.rag.inline
+
+        # If both BYOK and OKP are present and reranker is using default settings,
+        # ensure it's enabled for optimal results
+        if (
+            has_byok
+            and has_okp
+            and not self.reranker._explicitly_configured  # pylint: disable=protected-access
+            and not self.reranker.enabled
+        ):
+            logger.info(
+                "Automatically enabling reranker: Both BYOK RAG (%d entries) or "
+                "other inline RAG and OKP are configured. Reranking improves result "
+                "quality when multiple knowledge sources are available.",
+                len(self.byok_rag),
+            )
+            self.reranker.enabled = True
+
+        return self
+
     def dump(self, filename: str | Path = "configuration.json") -> None:
         """
         Write the current Configuration model to a JSON file.