Skip to content

Monitoring Results

glide.mean_monitoring_results.base

MeanMonitoringResult dataclass

Result of a drift-monitoring procedure over a batched dataset.

Holds the embedded confidence_sequence and exposes its per-look arrays as properties. Every array is indexed by batch in chronological order and is expressed in the original metric units. For batch t:

  • running_means[t] is the mean of the per-batch estimates up to t;
  • confidence_bounds[t] is the anytime-valid bound on that running mean, on the side where drift is harmful (a lower bound when the metric is a risk, an upper bound when it is a performance);
  • alarms[t] is True once confidence_bounds[t] has crossed alarm_threshold, signalling a drift.

alarm_threshold is the user-supplied business threshold the confidence bound is tested against: the worst metric value the user is willing to tolerate, in metric units.

Because the threshold is a known constant, the only false-alarm budget is the embedded sequence's, reported as confidence_level: under no drift, the probability of ever raising a false alarm is at most 1 - confidence_level.

Source code in glide/mean_monitoring_results/base.py 
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
@dataclass(repr=False)
class MeanMonitoringResult:
    """Result of a drift-monitoring procedure over a batched dataset.

    Holds the embedded ``confidence_sequence`` and exposes its per-look arrays as
    properties. Every array is indexed by batch in chronological order and is
    expressed in the original metric units. For batch ``t``:

    - ``running_means[t]`` is the mean of the per-batch estimates up to ``t``;
    - ``confidence_bounds[t]`` is the anytime-valid bound on that running mean,
      on the side where drift is harmful (a lower bound when the metric is a
      risk, an upper bound when it is a performance);
    - ``alarms[t]`` is ``True`` once ``confidence_bounds[t]`` has crossed
      ``alarm_threshold``, signalling a drift.

    ``alarm_threshold`` is the user-supplied business threshold the confidence
    bound is tested against: the worst metric value the user is willing to
    tolerate, in metric units.

    Because the threshold is a known constant, the only false-alarm budget is
    the embedded sequence's, reported as ``confidence_level``: under no drift,
    the probability of ever raising a false alarm is at most
    ``1 - confidence_level``.
    """

    metric_name: str
    monitor_name: str
    higher_is_better: bool
    alarm_threshold: float
    confidence_level: float
    batch_mean_estimates: NDArray
    confidence_sequence: ConfidenceSequence

    def __post_init__(self) -> None:
        _validate_non_empty(self.batch_mean_estimates, "batch_mean_estimates")
        _validate_equal_lengths(
            self.batch_mean_estimates,
            self.confidence_sequence.running_mean_estimates,
            self.confidence_sequence.confidence_bounds,
            names=["batch_mean_estimates", "running_mean_estimates", "confidence_bounds"],
        )

    @property
    def running_means(self) -> NDArray:
        result = self.confidence_sequence.running_mean_estimates
        return result

    @property
    def confidence_bounds(self) -> NDArray:
        result = self.confidence_sequence.confidence_bounds
        return result

    @property
    def alarms(self) -> NDArray:
        alternative = "smaller" if self.higher_is_better else "larger"
        result = self.confidence_sequence.test_null_hypothesis(self.alarm_threshold, alternative)
        return result

    @property
    def n_batches(self) -> int:
        result = len(self.batch_mean_estimates)
        return result

    @property
    def drift_detected(self) -> bool:
        result = bool(self.alarms.any())
        return result

    @property
    def first_alarm_index(self) -> Optional[int]:
        if not self.drift_detected:
            return None
        result = int(self.alarms.argmax())
        return result

    def _common_lines(self) -> List[str]:
        lines = [
            f"Metric: {self.metric_name}",
            f"Monitor: {self.monitor_name}",
            f"Number of Batches: {self.n_batches}",
            f"Drift Detected: {self.drift_detected}",
            f"First Alarm Index: {self.first_alarm_index}",
            f"Alarm Threshold: {self.alarm_threshold:.3f}",
            f"Running Mean: {self.running_means[-1]:.3f}",
            f"Confidence Bound: {self.confidence_bounds[-1]:.3f}",
            f"Confidence Level: {self.confidence_level:.2f}",
        ]
        return lines

    def __str__(self) -> str:
        return "\n".join(self._common_lines())

    def summary(self) -> str:
        """Return a formatted summary of the monitoring result."""
        return self.__str__()

    def __repr__(self) -> str:
        return self.__str__()

summary 

summary()

Return a formatted summary of the monitoring result.

Source code in glide/mean_monitoring_results/base.py 
102
103
104
def summary(self) -> str:
    """Return a formatted summary of the monitoring result."""
    return self.__str__()

glide.mean_monitoring_results.classical

ClassicalMeanMonitoringResult dataclass

Bases: MeanMonitoringResult

Monitoring result for the classical (proxy-free) monitor.

Adds the per-batch labeled sample count to the shared base. batch_n[t] is the number of labeled samples in batch t and has the same length as batch_mean_estimates.

Source code in glide/mean_monitoring_results/classical.py 
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
@dataclass(repr=False)
class ClassicalMeanMonitoringResult(MeanMonitoringResult):
    """Monitoring result for the classical (proxy-free) monitor.

    Adds the per-batch labeled sample count to the shared base. ``batch_n[t]``
    is the number of labeled samples in batch ``t`` and has the same length as
    ``batch_mean_estimates``.
    """

    batch_n: NDArray

    def __post_init__(self) -> None:
        super().__post_init__()
        _validate_equal_lengths(self.batch_mean_estimates, self.batch_n, names=["batch_mean_estimates", "batch_n"])

    def __str__(self) -> str:
        lines = self._common_lines() + [
            f"batch_n: {self.batch_n.tolist()}",
        ]
        return "\n".join(lines)

glide.mean_monitoring_results.prediction_powered

PredictionPoweredMeanMonitoringResult dataclass

Bases: MeanMonitoringResult

Monitoring result for the prediction-powered monitor.

Adds the per-batch labeled and total sample counts to the shared base. batch_n_true[t] is the number of labeled samples in batch t and batch_n_proxy[t] the total number of samples (labeled plus unlabeled); both have the same length as batch_mean_estimates.

Source code in glide/mean_monitoring_results/prediction_powered.py 
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
@dataclass(repr=False)
class PredictionPoweredMeanMonitoringResult(MeanMonitoringResult):
    """Monitoring result for the prediction-powered monitor.

    Adds the per-batch labeled and total sample counts to the shared base.
    ``batch_n_true[t]`` is the number of labeled samples in batch ``t`` and
    ``batch_n_proxy[t]`` the total number of samples (labeled plus unlabeled);
    both have the same length as ``batch_mean_estimates``.
    """

    batch_n_true: NDArray
    batch_n_proxy: NDArray

    def __post_init__(self) -> None:
        super().__post_init__()
        _validate_equal_lengths(
            self.batch_mean_estimates,
            self.batch_n_true,
            self.batch_n_proxy,
            names=["batch_mean_estimates", "batch_n_true", "batch_n_proxy"],
        )

    def __str__(self) -> str:
        lines = self._common_lines() + [
            f"batch_n_true: {self.batch_n_true.tolist()}",
            f"batch_n_proxy: {self.batch_n_proxy.tolist()}",
        ]
        return "\n".join(lines)