"""mortgage-rate-10-year-spread.py — 30-year mortgage rate minus the 10-year Treasury yield, weekly since 1971.

Input : datasets/mortgage-rate-10-year-spread.csv (FRED MORTGAGE30US weekly %, DGS10 daily %),
        retrieved 2026-09-05 via the public fredgraph CSV. Re-download with --download.
Method: for each survey week (dated Thursday) take the DGS10 close on that date, or the most recent
        earlier close if the market was shut; spread = MORTGAGE30US - DGS10 in percentage points.
        The FRED series page notes a PMMS methodology change on 2022-11-17.
Run   : python code/mortgage-rate-10-year-spread.py [--download]
Needs : Python 3.13, pandas 3.0.2.
"""
import io
import sys
import urllib.request

import pandas as pd

CSV = "datasets/mortgage-rate-10-year-spread.csv"


def download():
    frames = []
    for sid in ("MORTGAGE30US", "DGS10"):
        req = urllib.request.Request(f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={sid}",
                                     headers={"User-Agent": "prism-data-lab/1.0"})
        txt = urllib.request.urlopen(req, timeout=60).read().decode()
        frames.append(pd.read_csv(io.StringIO(txt), na_values=".", index_col=0).iloc[:, 0].rename(sid))
    df = pd.concat(frames, axis=1)
    df.index.name = "date"
    df[df.index >= "1971-04-01"].to_csv(CSV)


def main():
    if "--download" in sys.argv:
        download()
    df = pd.read_csv(CSV, parse_dates=["date"], index_col="date")
    dgs = df["DGS10"].ffill()
    w = df["MORTGAGE30US"].dropna().to_frame("mort")
    w["dgs10"] = dgs.reindex(w.index)
    w = w.dropna().copy()
    w["spread"] = w["mort"] - w["dgs10"]
    w["decade"] = (w.index.year // 10) * 10
    print("| decade | weeks | mean spread (pp) | median | narrowest (week) | widest (week) |")
    print("|---|---|---|---|---|---|")
    for dec, s in w.groupby("decade")["spread"]:
        print(f"| {dec}s | {len(s)} | {s.mean():.2f} | {s.median():.2f} | {s.min():.2f} ({s.idxmin():%Y-%m-%d}) | "
              f"{s.max():.2f} ({s.idxmax():%Y-%m-%d}) |")
    print("\n| year | mean mortgage rate | mean 10-year yield | mean spread (pp) |")
    print("|---|---|---|---|")
    recent = w.loc["2018":]
    for y, s in recent.groupby(recent.index.year):
        print(f"| {y} | {s['mort'].mean():.2f} | {s['dgs10'].mean():.2f} | {s['spread'].mean():.2f} |")
    full = w["spread"]
    print(f"\nwhole sample {w.index[0]:%Y-%m-%d} to {w.index[-1]:%Y-%m-%d}: {len(w)} weeks, mean {full.mean():.2f}, "
          f"median {full.median():.2f}, widest {full.max():.2f} on {full.idxmax():%Y-%m-%d}, "
          f"narrowest {full.min():.2f} on {full.idxmin():%Y-%m-%d}")
    last = w.iloc[-1]
    print(f"latest week {w.index[-1]:%Y-%m-%d}: mortgage {last['mort']:.2f}, 10-year {last['dgs10']:.2f}, spread {last['spread']:.2f}")
    print(f"weeks with spread above 3.00 pp: {int((full > 3).sum())}; above 2.50: {int((full > 2.5).sum())}; "
          f"share of weeks since 2022-11-17 above 2.50: {100 * (w.loc['2022-11-17':, 'spread'] > 2.5).mean():.0f}%")
    print(f"mean spread 2010-2019: {w.loc['2010':'2019', 'spread'].mean():.2f}; 2023-2026: {w.loc['2023':, 'spread'].mean():.2f}")


if __name__ == "__main__":
    main()