Effect Size Explained: Cohen’s d, r and More, with Real Data

import numpy as np
import pandas as pd
from datetime import date

# The ScholarsColab Research Methods Index (papers level): anonymised, updated daily
DATA = "https://scholarscolab.com/wp-json/scholarscolab/v1/research-methods-data?level=papers"
papers = pd.read_csv(DATA, comment="#")   # lines starting with # hold the citation

def p_text(p):
    return "< 0.001" if p < 0.001 else f"= {p:.3f}"

print(papers.shape)
papers.head()

from scipy import stats
THIS_YEAR = date.today().year
w = papers[papers["year"].between(THIS_YEAR - 16, THIS_YEAR - 4)]

def effect_sizes(a, b):
    u, p = stats.mannwhitneyu(a, b, alternative="two-sided")
    la, lb = np.log(a + 1), np.log(b + 1)
    pooled = np.sqrt(((len(la) - 1) * la.var() + (len(lb) - 1) * lb.var()) / (len(la) + len(lb) - 2))
    return {"n A": len(a), "n B": len(b), "median ratio": a.median() / max(b.median(), 0.5),
            "Cohen's d (log)": (la.mean() - lb.mean()) / pooled,
            "rank-biserial r": 2 * u / (len(a) * len(b)) - 1, "p": p}

comparisons = {
    "IEEE Transactions vs other journals": (w.venue_group == "IEEE Transactions / Journals", w.venue_group == "Other journals"),
    "Open-access mega vs other journals": (w.venue_group == "Open-access mega-journals", w.venue_group == "Other journals"),
    "Review vs original": (w.review == 1, w.review == 0),
    "Journal vs conference": (w.venue_type == "journal", w.venue_type == "conference"),
    "Colon in title vs none": (w.title_colon == 1, w.title_colon == 0),
}
table = pd.DataFrame({name: effect_sizes(w.loc[a, "citations"], w.loc[b, "citations"])
                      for name, (a, b) in comparisons.items()}).T
table.round(3)

import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(8, 3.5))
ax.barh(table.index, table["rank-biserial r"], color="lightgreen", edgecolor="green")
for x in (0.1, 0.3, 0.5):
    ax.axvline(x, color="grey", linestyle=":")
ax.set_xlabel("Rank-biserial r (0.1 small, 0.3 medium, 0.5 large)")
ax.invert_yaxis()
plt.tight_layout()
plt.show()