ensemble output is now set to [0,1] range, same for inconsistency resolution input. mv and wmv-conf are now separate models

Author: sfluegel05

README.md

@@ -162,13 +162,15 @@ The two-sided scaling matters whenever a model's threshold is not 0.5: with $t_{
 negative prediction only has a range of $0.2$ to move in and a positive one a range of $0.8$, so
 without rescaling the positive side would systematically outweigh the negative side.
 
-Confidence can be disabled by the `use_confidence` parameter of the predict method (default: True).
+Confidence is used by the weighted voting ensembles (`wmv-conf` and `wmv-f1`). If the `ensemble_type`
+is set to `mv`, all votes count the same (confidence is fixed to 1), which gives an unweighted
+majority-voting baseline.
 
 The`model_weight` can be set for each model in the configuration file (default: 1). This is used to favor a certain
 model independently of a given class.
 `Trust` is based on the model's performance on a validation set. After training, we evaluate the Machine Learning models
 on a validation set for each class. If the `ensemble_type` is set to `wmv-f1`, the trust is calculated as F1-score $^{6.25}$.
-If the `ensemble_type` is set to `mv` (the default), the trust is set to 1 for all models.
+For `mv` and `wmv-conf`, the trust is set to 1 for all models.
 
 #### Learned aggregation (`ltr` and `des`)
 

chebifier/__init__.py

@@ -2,10 +2,16 @@
 # even if multiple subpackages are imported later.
 
 from ._custom_cache import PerSmilesPerModelLRUCache, modelwise_smiles_lru_cache
-from .ensemble.voting_ensemble import VotingEnsemble
+from .ensemble.voting_ensemble import (
+    MajorityVotingEnsemble,
+    VotingEnsemble,
+    WMVwithConfidenceEnsemble,
+)
 
 __all__ = [
     "VotingEnsemble",
+    "MajorityVotingEnsemble",
+    "WMVwithConfidenceEnsemble",
     "PerSmilesPerModelLRUCache",
     "modelwise_smiles_lru_cache",
 ]

chebifier/ensemble/base_ensemble.py

@@ -27,12 +27,13 @@ def ensemble_name(self):
 
     @property
     def decision_threshold(self):
-        """Value a net score has to exceed for the class to be predicted.
+        """Probability a class has to exceed to be predicted.
 
-        Zero for every ensemble whose scores are already centred on their decision boundary; only
-        ensembles that emit a calibrated, unshifted scale need to override this.
+        Ensembles report probabilities, so the default is the neutral point: predict a class when
+        the ensemble believes it more likely than not. Ensembles that tune their own operating
+        point (to maximise macro-F1, say) override this.
         """
-        return 0.0
+        return 0.5
 
     def calibrate(
         self,

chebifier/ensemble/dynamic_selection_ensemble.py

@@ -14,6 +14,7 @@
     cv_folds,
     dense_from_pairs,
     holdout_split,
+    noncandidate_probability,
     pair_scorer,
     rescale_to_threshold,
     save_hyperparameter_results,
@@ -289,9 +290,8 @@ def aggregate(delta, chunk, competence_threshold, vote):
     if vote == "confidence":
         direction = direction * 2.0 * np.abs(chunk.profiles - POSITIVE_THRESHOLD)
     total = weight.sum(axis=1)
-    return ((weight * direction).sum(axis=1) / np.maximum(total, 1e-6)).astype(
-        np.float32
-    )
+    agreement = (weight * direction).sum(axis=1) / np.maximum(total, 1e-6)
+    return np.clip(0.5 + agreement / 2, 0.0, 1.0).astype(np.float32)
 
 
 class DynamicSelectionEnsemble(VotingEnsemble):
@@ -407,6 +407,11 @@ def calibrate(self, validation_predictions, validation_data, validation_labels):
             labels, candidates.molecule, candidates.class_index, dev_mask
         )
         tau, dev_macro_f1 = scorer.tune(net[keep])
+        floor = noncandidate_probability(
+            scores.shape[0] * scores.shape[1] - candidates.molecule.size,
+            int(labels.sum())
+            - int(labels[candidates.molecule, candidates.class_index].sum()),
+        )
 
         # the meta-classifier and tau are fitted with the dev molecules held out of the reference
         # set, but nothing stops prediction from looking neighbours up in all of them
@@ -433,10 +438,10 @@ def calibrate(self, validation_predictions, validation_data, validation_labels):
                 "consensus_threshold": float(self.consensus_threshold),
                 "competence_threshold": float(self.competence_threshold),
                 "tau": float(tau),
+                "floor": floor,
                 "n_classes": int(scores.shape[1]),
                 "n_dsel": int(reference_mask.sum()),
                 "dev_macro_f1": float(dev_macro_f1),
-                "normalized": True,
             },
         )
         print(
@@ -732,12 +737,11 @@ def _load(self):
             )
         with open(self._metadata_path, "r", encoding="utf-8") as f:
             self._metadata = json.load(f)
-        if not self._metadata.get("normalized"):
+        if "floor" not in self._metadata:
             raise ValueError(
-                f"The ensemble in {self.ensemble_dir} was calibrated before net scores were "
-                "normalised by the selection weight. Its stored tau is on the old (unnormalised) "
-                "scale and would reject every prediction. Please re-run `chebifier build` for this "
-                "ensemble."
+                f"The ensemble in {self.ensemble_dir} was calibrated before scores became "
+                "probabilities. Its stored tau is on the old scale and would reject every "
+                "prediction. Please re-run `chebifier build` for this ensemble."
             )
         with open(self._classifier_path, "rb") as f:
             self._classifier = pickle.load(f)
@@ -809,10 +813,17 @@ def predict(self, test_predictions, molecules=None):
         dense = dense_from_pairs(
             candidates.molecule,
             candidates.class_index,
-            net - self._metadata["tau"],
+            net,
             scores.shape[:2],
+            floor=self._metadata["floor"],
         )
         return {
             "net_score": torch.from_numpy(dense),
             "has_valid_predictions": torch.from_numpy((~np.isnan(scores)).any(axis=2)),
         }
+
+    @property
+    def decision_threshold(self):
+        """Probability a candidate has to exceed, the operating point macro-F1 peaks at."""
+        self._load()
+        return float(self._metadata["tau"])

chebifier/ensemble/learning_to_rank_ensemble.py

@@ -13,7 +13,7 @@
     dense_from_pairs,
     fit_platt,
     holdout_split,
-    noncandidate_log_odds,
+    noncandidate_probability,
     pair_scorer,
     save_hyperparameter_results,
     select_candidates,
@@ -75,6 +75,10 @@ def build_rows(scores, k, labels=None):
     return rows
 
 
+def _sigmoid(x):
+    return float(1.0 / (1.0 + np.exp(-x)))
+
+
 def design_matrix(rows, class_stats=None):
     if class_stats is None:
         return rows["X"]
@@ -142,7 +146,7 @@ def calibrate(self, validation_predictions, validation_data, validation_labels):
             rows, labels, train_mask, dev_mask, scores, thresholds
         )
         platt_a, platt_b = fit_platt(dev_scores, rows["y"][dev_mask[rows["molecule"]]])
-        floor_logodds = noncandidate_log_odds(
+        floor = noncandidate_probability(
             scores.shape[0] * scores.shape[1] - rows["molecule"].size,
             int(labels.sum()) - int(rows["y"].sum()),
         )
@@ -161,8 +165,8 @@ def calibrate(self, validation_predictions, validation_data, validation_labels):
             "tau": float(tau),
             "platt_a": platt_a,
             "platt_b": platt_b,
-            "tau_logodds": platt_a * float(tau) + platt_b,
-            "floor_logodds": floor_logodds,
+            "tau_probability": _sigmoid(platt_a * float(tau) + platt_b),
+            "floor": floor,
             "n_classes": int(scores.shape[1]),
             "best_iteration": int(booster.best_iteration),
             "dev_macro_f1": float(dev_macro_f1),
@@ -175,9 +179,9 @@ def calibrate(self, validation_predictions, validation_data, validation_labels):
         self._booster, self._metadata = booster, metadata
         print(
             f"Saved ranker to {self._model_path} (candidate_k={candidate_k}, tau={tau:.4f}, "
-            f"held-out macro-f1: {dev_macro_f1:.4f}). Calibrated to log-odds with "
-            f"a={platt_a:.4f}, b={platt_b:.4f} (decision threshold {metadata['tau_logodds']:.4f}); "
-            f"non-candidate pairs scored at {floor_logodds:.2f}."
+            f"held-out macro-f1: {dev_macro_f1:.4f}). Calibrated to probabilities with "
+            f"a={platt_a:.4f}, b={platt_b:.4f} (decision threshold "
+            f"{metadata['tau_probability']:.4f}); non-candidate pairs scored at {floor:.2e}."
         )
 
     def _fit(self, rows, labels, train_mask, dev_mask, scores=None, thresholds=None):
@@ -299,13 +303,9 @@ def _load(self):
 
     @property
     def decision_threshold(self):
-        """Decision threshold on the scale `predict` returns.
-
-        Zero for rankers calibrated before Platt scaling was introduced - those still subtract tau
-        from the score themselves.
-        """
+        """Probability a candidate has to exceed, the operating point macro-F1 peaks at."""
         self._load()
-        return float(self._metadata.get("tau_logodds", 0.0))
+        return float(self._metadata["tau_probability"])
 
     def predict(self, test_predictions, molecules=None):
         self._load()
@@ -314,15 +314,18 @@ def predict(self, test_predictions, molecules=None):
         raw = self._booster.predict(design_matrix(rows, self._class_stats)).astype(
             np.float32
         )
-        if "platt_a" in self._metadata:
-            # calibrated log-odds, with the decision threshold left to the caller so that the
-            # inconsistency resolution sees an unshifted scale
-            net = self._metadata["platt_a"] * raw + self._metadata["platt_b"]
-            floor = self._metadata["floor_logodds"]
-        else:
-            net, floor = raw - self._metadata["tau"], None
+        # the ranker score is not a probability, so it goes through the calibration fitted during
+        # `calibrate`; the decision threshold is left to the caller so that inconsistency
+        # resolution sees the probabilities themselves
+        net = 1.0 / (
+            1.0 + np.exp(-(self._metadata["platt_a"] * raw + self._metadata["platt_b"]))
+        )
         dense = dense_from_pairs(
-            rows["molecule"], rows["class_index"], net, scores.shape[:2], floor=floor
+            rows["molecule"],
+            rows["class_index"],
+            net.astype(np.float32),
+            scores.shape[:2],
+            floor=self._metadata["floor"],
         )
         return {
             "net_score": torch.from_numpy(dense),

chebifier/ensemble/level1.py

@@ -165,13 +165,15 @@ def dense_from_pairs(molecule, class_index, net, shape, floor=None):
 
 
 def fit_platt(scores, labels):
-    """Fit `P(positive) = sigmoid(a * score + b)` so that `a * score + b` is calibrated log-odds.
+    """Fit `P(positive) = sigmoid(a * score + b)`, turning a raw score into a probability.
 
     A lambdarank score is only trained to order pairs within a group, so its scale carries no
-    probabilistic meaning. Inconsistency resolution needs one: it compares scores across classes and
-    maps them through `sigmoid(k * score)`. Being a monotone map, this leaves the ensemble's own
-    thresholded decisions untouched - what it buys is a scale on which those downstream comparisons
-    are meaningful.
+    probabilistic meaning. Inconsistency resolution needs one: it compares scores across classes.
+    Being a monotone map, this leaves the ensemble's own thresholded decisions untouched - what it
+    buys is a scale on which those downstream comparisons are meaningful.
+
+    The voting ensembles need no equivalent: their weighted agreement fraction is already a well
+    calibrated probability, and a logistic fit measurably degrades it.
     """
     from sklearn.linear_model import LogisticRegression
 
@@ -187,17 +189,16 @@ def fit_platt(scores, labels):
     return slope, intercept
 
 
-def noncandidate_log_odds(n_noncandidate, n_missed_positives):
-    """Log-odds that a pair outside the candidate set is nevertheless a positive.
+def noncandidate_probability(n_noncandidate, n_missed_positives):
+    """Probability that a pair outside the candidate set is nevertheless a positive.
 
     Candidate selection keeps only the top-k classes per model, so the pairs it drops are not
     "unknown" - they are overwhelmingly true negatives, and how overwhelmingly is measurable on the
     validation set. This turns the fill value for those pairs from an arbitrary floor into a
-    calibrated score that can be compared against the candidates. The Jeffreys-style pseudo-count
-    keeps the result finite when no positive is missed at all.
+    calibrated probability that can be compared against the candidates. The Jeffreys-style
+    pseudo-count keeps the result non-zero when no positive is missed at all.
     """
-    p = (n_missed_positives + 0.5) / (n_noncandidate + 1.0)
-    return float(np.log(p / (1.0 - p)))
+    return float((n_missed_positives + 0.5) / (n_noncandidate + 1.0))
 
 
 def save_hyperparameter_results(ensemble_dir, results, best):

chebifier/ensemble/voting_ensemble.py

@@ -8,13 +8,15 @@
 
 
 class VotingEnsemble(BaseEnsemble):
+    """Base class for the voting ensembles. Subclasses define the vote weights by overriding
+    calculate_confidence (self-reported confidence of a model) and calculate_trust (measured
+    reliability of a model)."""
+
     def __init__(
         self,
         ensemble_dir: str,
-        use_confidence: bool = True,
     ):
         super().__init__(ensemble_dir)
-        self.use_confidence = use_confidence
         self.classwise_f1 = None
         self.prediction_thresholds = None
 
@@ -66,19 +68,26 @@ def _load_prediction_thresholds(self) -> dict[str, float]:
                 f"Prediction thresholds file not found in ensemble directory: {self.ensemble_dir}. Please calibrate the ensemble first."
             )
 
+    def calculate_confidence(
+        self, predictions_tensor: torch.Tensor, thresholds: torch.Tensor
+    ) -> torch.Tensor:
+        raise NotImplementedError
+
     def calculate_trust(self, predictions: dict[str, torch.Tensor]) -> torch.Tensor:
-        # No trust for MV, only used in WMV
+        # No trust unless a subclass measures it (e.g. WMVwithF1Ensemble)
         return 1
 
     def predict(self, test_predictions: dict[str, torch.Tensor], molecules=None):
         """
-        Aggregates predictions from multiple models using weighted majority voting.
-        weights are only the self-reported confidence (=difference between prediction and threshold). If set to false, all models are weighted equally.
-
-        The net score is normalised by the weight mass that was cast, so it is a signed agreement
-        fraction in [-1, 1] rather than a sum over models. This keeps classes covered by different
-        numbers of base learners comparable, which matters downstream: inconsistency resolution
-        compares scores across classes. The sign is unaffected, so class decisions do not change.
+        Aggregates predictions from multiple models by voting, each vote weighted by
+        calculate_confidence * calculate_trust.
+
+        The net score is the weighted agreement among the models that voted, mapped onto [0, 1]:
+        1 if they unanimously predict the class, 0 if they unanimously reject it, 0.5 if they are
+        evenly split. Normalising by the weight mass that was cast (rather than summing over models)
+        keeps classes covered by different numbers of base learners comparable, which is what
+        inconsistency resolution needs - it compares scores across classes. The agreement fraction
+        is a well calibrated probability on its own, so no further calibration is applied.
         """
         predictions_tensor = torch.stack(
             list(test_predictions.values()), dim=2
@@ -114,16 +123,9 @@ def predict(self, test_predictions: dict[str, torch.Tensor], molecules=None):
             predictions_tensor < threshold_mask.unsqueeze(0).unsqueeze(0)
         ) & valid_predictions
 
-        if self.use_confidence:
-            threshold = threshold_mask.unsqueeze(0).unsqueeze(0)
-            scores = predictions_tensor.nan_to_num()
-            confidence = torch.where(
-                scores < threshold,
-                (threshold - scores) / threshold,
-                (scores - threshold) / (1 - threshold),
-            )
-        else:
-            confidence = torch.ones_like(predictions_tensor)
+        confidence = self.calculate_confidence(
+            predictions_tensor, threshold_mask.unsqueeze(0).unsqueeze(0)
+        )
 
         trust = self.calculate_trust(test_predictions)
         # Calculate weighted predictions using broadcasting
@@ -141,8 +143,13 @@ def predict(self, test_predictions: dict[str, torch.Tensor], molecules=None):
         )  # Shape: (num_molecules, num_classes)
 
         # Determine which classes to include for each molecule
-        net_score = (positive_sum - negative_sum) / (positive_sum + negative_sum).clamp(
-            min=1e-6
+        net_score = (
+            0.5
+            + (positive_sum - negative_sum)
+            / (positive_sum + negative_sum).clamp(min=1e-6)
+            / 2
+        ).clamp(
+            0.0, 1.0
         )  # Shape: (num_molecules, num_classes)
         return {
             "net_score": net_score,
@@ -153,3 +160,28 @@ def predict(self, test_predictions: dict[str, torch.Tensor], molecules=None):
             "positive_mask": positive_mask,
             "negative_mask": negative_mask,
         }
+
+
+class MajorityVotingEnsemble(VotingEnsemble):
+    """Plain majority voting: every model that votes counts the same, no weights at all."""
+
+    def calculate_confidence(
+        self, predictions_tensor: torch.Tensor, thresholds: torch.Tensor
+    ) -> torch.Tensor:
+        return torch.ones_like(predictions_tensor)
+
+
+class WMVwithConfidenceEnsemble(VotingEnsemble):
+    """WMV ensemble that weights each vote by the model's self-reported confidence, i.e. how far
+    its prediction sits from its decision threshold, scaled separately on each side so that a
+    maximally confident negative and a maximally confident positive both count 1."""
+
+    def calculate_confidence(
+        self, predictions_tensor: torch.Tensor, thresholds: torch.Tensor
+    ) -> torch.Tensor:
+        scores = predictions_tensor.nan_to_num()
+        return torch.where(
+            scores < thresholds,
+            (thresholds - scores) / thresholds,
+            (scores - thresholds) / (1 - thresholds),
+        )