ranklens.viz

ranklens.viz

Charts of results (plotly, extra viz).

attribution_heatmap

attribution_heatmap(
    diagnoses: Sequence[QueryDiagnosis],
) -> Figure

Features by the worst queries: red where shuffling a feature helps (it misleads).

Source code in src/ranklens/viz/charts.py

def attribution_heatmap(diagnoses: Sequence[QueryDiagnosis]) -> Figure:
    """Features by the worst queries: red where shuffling a feature helps (it misleads)."""
    go = _plotly()
    features = sorted({label for d in diagnoses for label, _ in d.causes})
    z = [[dict(d.causes).get(f, 0.0) for d in diagnoses] for f in features]
    figure = go.Figure(
        go.Heatmap(
            z=z,
            x=[str(d.query_id) for d in diagnoses],
            y=features,
            colorscale="RdBu",
            zmid=0.0,
        )
    )
    metric = diagnoses[0].metric if diagnoses else "metric"
    return figure.update_layout(
        xaxis_title="query (worst first)", yaxis_title=f"drop of {metric} when shuffled"
    )

delta_distribution

delta_distribution(
    baseline: MetricResult, candidate: MetricResult
) -> Figure

Histogram of per-query differences: the tails matter more than the mean.

Source code in src/ranklens/viz/charts.py

def delta_distribution(baseline: MetricResult, candidate: MetricResult) -> Figure:
    """Histogram of per-query differences: the tails matter more than the mean."""
    go = _plotly()
    common = sorted(baseline.per_query.keys() & candidate.per_query.keys())
    deltas = [candidate.per_query[q] - baseline.per_query[q] for q in common]
    figure = go.Figure(go.Histogram(x=deltas, nbinsx=40, name="queries"))
    return figure.update_layout(
        xaxis_title=f"{baseline.metric}: B - A per query", yaxis_title="queries"
    )

metric_vs_k

metric_vs_k(
    evaluations: Mapping[str, Evaluation],
    metric: str = "ndcg",
) -> Figure

One line per run: the mean of metric@k for every cutoff the evaluations hold.

Source code in src/ranklens/viz/charts.py

def metric_vs_k(evaluations: Mapping[str, Evaluation], metric: str = "ndcg") -> Figure:
    """One line per run: the mean of ``metric@k`` for every cutoff the evaluations hold."""
    go = _plotly()
    figure = go.Figure()
    for run, evaluation in evaluations.items():
        points = sorted(
            (spec.k, result.mean)
            for result in evaluation.metrics
            if (spec := parse_spec(result.metric)).name == metric and spec.k is not None
        )
        if not points:
            raise ValueError(f"run {run!r} has no {metric}@k results")
        figure.add_trace(go.Scatter(x=[k for k, _ in points], y=[m for _, m in points], name=run))
    return figure.update_layout(xaxis_title="k", yaxis_title=f"{metric}@k")

propensity_diagnostics

propensity_diagnostics(
    diagnostics: WeightDiagnostics,
) -> Figure

Distribution of the weights 1/p, with the effective sample size in the title.

Source code in src/ranklens/viz/charts.py

def propensity_diagnostics(diagnostics: WeightDiagnostics) -> Figure:
    """Distribution of the weights 1/p, with the effective sample size in the title."""
    go = _plotly()
    figure = go.Figure(go.Histogram(x=list(diagnostics.weights), nbinsx=40))
    title = (
        f"ESS {diagnostics.ess:.0f} of {diagnostics.n_weights} weights "
        f"({diagnostics.ess_share:.0%}); max {diagnostics.max_weight:.1f}"
    )
    return figure.update_layout(title=title, xaxis_title="weight 1/p", yaxis_title="rewards")

rank_flow

rank_flow(
    pairs: Iterable[tuple[RankedList, RankedList]],
    k: int = 10,
) -> Figure

Where documents of A's top k end up in B (and where B's top comes from).

Source code in src/ranklens/viz/charts.py

def rank_flow(pairs: Iterable[tuple[RankedList, RankedList]], k: int = 10) -> Figure:
    """Where documents of A's top ``k`` end up in B (and where B's top comes from)."""
    if k < 1:
        raise ValueError(f"k must be >= 1, got {k}")
    go = _plotly()
    flows: Counter[tuple[str, str]] = Counter()
    for a, b in pairs:
        position_b = {doc: r for r, doc in enumerate(b.docs, 1)}
        position_a = {doc: r for r, doc in enumerate(a.docs, 1)}
        for doc in set(a.docs[:k]) | set(b.docs[:k]):
            flows[_bucket(position_a.get(doc), k), _bucket(position_b.get(doc), k)] += 1
    buckets = _buckets(k)
    nodes = [f"A {b}" for b in buckets] + [f"B {b}" for b in buckets]
    index = {name: i for i, name in enumerate(nodes)}
    links = [(index[f"A {src}"], index[f"B {dst}"], n) for (src, dst), n in sorted(flows.items())]
    return go.Figure(
        go.Sankey(
            node={"label": nodes},
            link={
                "source": [s for s, _, _ in links],
                "target": [t for _, t, _ in links],
                "value": [n for _, _, n in links],
            },
        )
    )

segment_gains

segment_gains(
    comparisons: Mapping[str, ComparisonResult],
) -> Figure

Difference per segment with its interval; significant ones (after BH) stand out.

Source code in src/ranklens/viz/charts.py

def segment_gains(comparisons: Mapping[str, ComparisonResult]) -> Figure:
    """Difference per segment with its interval; significant ones (after BH) stand out."""
    go = _plotly()
    labels = list(comparisons)
    results = [comparisons[label] for label in labels]
    colors = ["#1f77b4" if r.significant else "#b0b0b0" for r in results]
    figure = go.Figure(
        go.Bar(
            x=[r.delta for r in results],
            y=labels,
            orientation="h",
            marker_color=colors,
            error_x={
                "type": "data",
                "symmetric": False,
                "array": [r.ci_high - r.delta for r in results],
                "arrayminus": [r.delta - r.ci_low for r in results],
            },
            # q after the correction, p before it: the hover names which one it shows
            customdata=[
                f"q = {r.q_value:.4f}" if r.q_value is not None else f"p = {r.p_value:.4f}"
                for r in results
            ],
            hovertemplate="%{y}: %{x:+.4f}, %{customdata}<extra></extra>",
        )
    )
    return figure.update_layout(xaxis_title="difference B - A", yaxis_title="segment")