How the scoring rule shaped the ranking

from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

import plotstyle
from plotstyle import tidy

REPO = Path("..")
EXTRA = REPO / "analysis" / "2026-10-membership-rule" / "data"


def read(path):
    # keep_default_na=False: "NA" is Namibia's country code, not a missing value
    return pd.read_csv(path, keep_default_na=False)


september = read(REPO / "youtube-monetization-signals-by-country.csv")
october = read(EXTRA / "country-monetization-recomputed-2026-10-19.csv")
memberships = read(EXTRA / "youtube-channel-memberships-locations-2026-10-20.csv")
membership_codes = set(memberships["iso2"])
# YouTube Help listed 100 locations on 20 October 2026
assert len(membership_codes) == 100
assert len(september) == 50  # the text says "50 countries"


def add_shares(table):
    table = table.copy()
    table["memberships"] = table["country_code"].isin(membership_codes)
    table["likely_share"] = table["likely_monetized"] / table["channels_checked"]
    return table


september = add_shares(september)
october = add_shares(october)
print(f"September: {len(september)} countries; "
      f"October: {len(october)} countries, "
      f"{october['channels_checked'].sum():,} channels")

# Same order as the previous chapter: by likely share, ties broken by sample size
ranked = september.sort_values(["likely_share", "channels_checked"]).iloc[::-1]
ranked = ranked.reset_index(drop=True)
without = ranked[~ranked["memberships"]]
with_memberships = ranked[ranked["memberships"]]
bottom = set(ranked.tail(len(without))["country_code"])
assert bottom == set(without["country_code"])
assert len(without) == 8  # the text of this chapter says "eight"
likely_without = int(without["likely_monetized"].sum())
checked_without = int(without["channels_checked"].sum())
WORDS = "zero one two three four five six seven eight nine ten".split()  # for the text

lowest_share = with_memberships["likely_share"].min()
lowest_with = with_memberships[with_memberships["likely_share"] == lowest_share]
lowest_label = (
    "The lowest country that does have memberships"
    if len(lowest_with) == 1
    else "The lowest countries that do have memberships"
)
print("without memberships:", ", ".join(without["country"]))
print(f"best of them {without['likely_share'].max():.1%}; "
      f"lowest with memberships: {' and '.join(lowest_with['country'])} "
      f"at {lowest_share:.1%}")

counts = ["channels_checked", "likely_monetized"]
totals = september.groupby("memberships")[counts].sum()
pooled_share = totals["likely_monetized"] / totals["channels_checked"]

# How much of the variation in the likely share across countries the yes/no
# memberships split accounts for (1 minus within-group / total sum of squares)
share = september["likely_share"]
group_mean = september.groupby("memberships")["likely_share"].transform("mean")
within_groups = ((share - group_mean) ** 2).sum()
overall = ((share - share.mean()) ** 2).sum()
explained = 1 - within_groups / overall
print(f"pooled likely share: with {pooled_share[True]:.1%}, "
      f"without {pooled_share[False]:.1%}; explained {explained:.1%}")

both = september.merge(october, on="country_code", suffixes=("_sep", "_oct"))
both["channel_change"] = (
    both["channels_checked_oct"] - both["channels_checked_sep"]
)
same_sample = both[both["channel_change"].abs() <= 6].copy()
same_sample["rise"] = (
    same_sample["likely_share_oct"] - same_sample["likely_share_sep"]
)

by_group = same_sample.groupby("memberships_sep")[
    ["likely_share_sep", "likely_share_oct", "rise"]
].mean()
same_without = int((~same_sample["memberships_sep"]).sum())


def rank_correlation(a, b):
    """Spearman's rho: Pearson on average ranks, so no SciPy is needed."""
    return float(np.corrcoef(pd.Series(a).rank(), pd.Series(b).rank())[0, 1])


rho_same = rank_correlation(
    same_sample["likely_share_sep"], same_sample["likely_share_oct"]
)
iraq = both.set_index("country_code").loc["IQ"]
tanzania = both.set_index("country_code").loc["TZ"]
group_names = {True: "memberships offered", False: "not offered"}
by_group.rename(index=group_names).style.format("{:.1%}")

october_ranked = october.sort_values("likely_share", ascending=False)
october_ranked = october_ranked.reset_index(drop=True)
october_ranked["rank"] = october_ranked.index + 1
ranks_without = october_ranked.loc[~october_ranked["memberships"], "rank"]
top10_september = set(ranked.head(10)["country_code"])
top10_october = set(october_ranked.head(10)["country_code"])
top10_both = len(top10_september & top10_october)
print(f"October ranks of the eight: {ranks_without.min()} to "
      f"{ranks_without.max()} of {len(october)}; top-10 overlap {top10_both}; "
      f"rank correlation (same sample) {rho_same:.2f}")