ranklens.aggregate

ranklens.aggregate

Slices of a comparison: the same metric within segments of queries.

check_same_segments

check_same_segments(
    baseline: Mapping[QueryId, SegmentKey],
    candidate: Mapping[QueryId, SegmentKey],
) -> None

Warn when a query has different segment values in the two runs.

A segment describes the query, not the model, so both runs should agree; when they do not, callers slice by the baseline and the warning says so.

Source code in src/ranklens/aggregate/segments.py

def check_same_segments(
    baseline: Mapping[QueryId, SegmentKey], candidate: Mapping[QueryId, SegmentKey]
) -> None:
    """Warn when a query has different segment values in the two runs.

    A segment describes the query, not the model, so both runs should agree; when they
    do not, callers slice by the baseline and the warning says so.
    """
    differ = sorted(q for q in baseline.keys() & candidate.keys() if baseline[q] != candidate[q])
    if differ:
        first = differ[0]
        warnings.warn(
            f"{len(differ)} queries have different segment values in the two runs, "
            f"e.g. {first!r}: {baseline[first]} and {candidate[first]}; "
            "the baseline's values are used",
            RankLensWarning,
            stacklevel=3,
        )

compare_segments

compare_segments(
    baseline: MetricResult,
    candidate: MetricResult,
    segments: Mapping[QueryId, SegmentKey],
    names: Sequence[str],
    *,
    min_queries: int = MIN_QUERIES,
    alpha: float = 0.05,
    n_resamples: int = 10000,
    n_permutations: int = 10000,
    seed: int = 0,
) -> pd.DataFrame

One row per segment: difference, interval, p-value and BH-corrected q-value.

Segments with fewer than min_queries paired queries are listed with empty statistics instead of unreliable ones — they are shown, not silently dropped, and they do not take part in the correction. Rows are sorted by the segment fields.

Each segment draws its own random stream from seed and its own values: with one shared stream, segments of equal size would get identical resamples and correlated p-values; and a segment keeps its numbers when other segments come or go. Paired queries without segment values are left out with a warning.

Source code in src/ranklens/aggregate/segments.py

def compare_segments(
    baseline: MetricResult,
    candidate: MetricResult,
    segments: Mapping[QueryId, SegmentKey],
    names: Sequence[str],
    *,
    min_queries: int = MIN_QUERIES,
    alpha: float = 0.05,
    n_resamples: int = 10_000,
    n_permutations: int = 10_000,
    seed: int = 0,
) -> pd.DataFrame:
    """One row per segment: difference, interval, p-value and BH-corrected q-value.

    Segments with fewer than ``min_queries`` paired queries are listed with empty
    statistics instead of unreliable ones — they are shown, not silently dropped, and
    they do not take part in the correction. Rows are sorted by the segment fields.

    Each segment draws its own random stream from ``seed`` and its own values: with one
    shared stream, segments of equal size would get identical resamples and correlated
    p-values; and a segment keeps its numbers when other segments come or go.
    Paired queries without segment values are left out with a warning.
    """
    _check_names(names)
    groups, n_unplaced = _group(baseline, candidate, segments, names)
    if n_unplaced:
        warnings.warn(
            f"{n_unplaced} paired queries have no value for {list(names)} and are in no segment",
            RankLensWarning,
            stacklevel=2,
        )
    compared: dict[SegmentKey, ComparisonResult] = {}
    small: dict[SegmentKey, int] = {}
    for key in sorted(groups):
        queries = groups[key]
        if len(queries) < min_queries:
            small[key] = len(queries)
            continue
        compared[key] = compare(
            _subset(baseline, queries),
            _subset(candidate, queries),
            min_queries=min_queries,
            alpha=alpha,
            n_resamples=n_resamples,
            n_permutations=n_permutations,
            seed=_stream(seed, key),
        )
    corrected = dict(zip(compared, adjust(list(compared.values())), strict=True))

    rows = [
        {**dict(zip(names, key, strict=True)), **_row(result)} for key, result in corrected.items()
    ]
    rows += [
        {**dict(zip(names, key, strict=True)), "n_queries": n_queries}
        for key, n_queries in small.items()
    ]
    frame = pd.DataFrame(rows, columns=[*names, *COLUMNS])
    return frame.sort_values(by=list(names), ignore_index=True)

segment_values

segment_values(
    result: MetricResult,
    segments: Mapping[QueryId, SegmentKey],
    names: Sequence[str],
) -> pd.DataFrame

Per-query values with their segment fields, one row per query.

Source code in src/ranklens/aggregate/segments.py

def segment_values(
    result: MetricResult, segments: Mapping[QueryId, SegmentKey], names: Sequence[str]
) -> pd.DataFrame:
    """Per-query values with their segment fields, one row per query."""
    _check_names(names)
    rows = [
        {
            **dict(zip(names, segments.get(query, ()), strict=False)),
            "query_id": query,
            "value": value,
        }
        for query, value in result.per_query.items()
    ]
    return pd.DataFrame(rows, columns=[*names, "query_id", "value"])