"""free-market-data-sources-and-lags.py — for eight FRED series, measure the native frequency and how far the last observation sits behind today.

Input : datasets/free-market-data-sources-and-lags.csv (FRED DGS10, CPIAUCSL, MORTGAGE30US, FEDFUNDS,
        UNRATE, GDPC1, VIXCLS since 2025-01-01), retrieved 2026-09-05 via the public fredgraph CSV.
        Re-download with --download.
        The eighth series, SP500, is NOT shipped in the CSV. S&P Dow Jones Indices LLC states in the FRED
        series notes that "Reproduction of S&P 500 in any form is prohibited except with the prior written
        permission of S&P Dow Jones Indices LLC ("S&P")". Its row in the table below is reported from the
        published 2026-09-05 retrieval as five quoted facts about the series - the count of observations in
        the window, their spacing, the last observation date, its age, and that day's close - which is
        quotation, not redistribution. Source: S&P Dow Jones Indices LLC,
        https://fred.stlouisfed.org/series/SP500. Run with --download to pull SP500 from FRED and print
        today's row for it instead, computed live and never written to disk.
Method: for each column, count non-missing observations, infer the median spacing between them (days),
        take the last observation date, and compute the lag in days from the retrieval date. The
        'observation lag' is the gap between the period a number describes and the day you can read it;
        it is not the same as revision risk, which this script does not measure.
Run   : python code/free-market-data-sources-and-lags.py [--download]
Needs : Python 3.13, pandas 3.0.2.
"""
import datetime as dt
import io
import sys
import urllib.request

import pandas as pd

CSV = "datasets/free-market-data-sources-and-lags.csv"
RETRIEVED = dt.date(2026, 9, 5)
SERIES = ("DGS10", "CPIAUCSL", "MORTGAGE30US", "FEDFUNDS", "UNRATE", "GDPC1", "VIXCLS")
# the S&P 500 row as published on the retrieval date; quoted, not shipped (see the header note)
SP500_ROW = {"observations": 420, "spacing": 1, "last": dt.date(2026, 9, 4), "value": 7718.60}
SOURCE = {"SP500": "S&P Dow Jones Indices via FRED (10-year window)", "DGS10": "Federal Reserve H.15",
          "CPIAUCSL": "BLS Consumer Price Index", "MORTGAGE30US": "Freddie Mac PMMS (weekly, Thursday)",
          "FEDFUNDS": "Federal Reserve H.15 (monthly average)", "UNRATE": "BLS Employment Situation",
          "GDPC1": "BEA GDP (quarterly, SAAR)", "VIXCLS": "Cboe via FRED"}


def fetch(sid):
    req = urllib.request.Request(f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={sid}",
                                 headers={"User-Agent": "prism-data-lab/1.0", "Accept": "*/*"})
    txt = urllib.request.urlopen(req, timeout=60).read().decode()
    return pd.read_csv(io.StringIO(txt), na_values=".", index_col=0).iloc[:, 0].rename(sid)


def download():
    df = pd.concat([fetch(sid) for sid in SERIES], axis=1)
    df.index.name = "date"
    df[df.index >= "2025-01-01"].to_csv(CSV)


def sp500_row(live: bool):
    """The S&P 500 row: quoted from the published retrieval, or computed live from FRED with --download."""
    if not live:
        d = SP500_ROW
        return d["observations"], d["spacing"], d["last"], (RETRIEVED - d["last"]).days, d["value"]
    s = fetch("SP500")
    s = s[s.index >= "2025-01-01"].dropna()
    idx = pd.to_datetime(s.index)
    last = idx[-1].date()
    return (len(s), idx.to_series().diff().dt.days.median(), last,
            (dt.date.today() - last).days, float(s.iloc[-1]))


def main():
    if "--download" in sys.argv:
        download()
    df = pd.read_csv(CSV, parse_dates=["date"], index_col="date")
    print(f"retrieved {RETRIEVED.isoformat()}; file covers {df.index[0]:%Y-%m-%d} to {df.index[-1]:%Y-%m-%d}")
    print("| series | source | observations in file | median spacing (days) | last observation | days behind retrieval | last value |")
    print("|---|---|---|---|---|---|---|")
    n, sp, last, behind, val = sp500_row("--download" in sys.argv)
    print(f"| SP500 | {SOURCE['SP500']} | {n} | {sp:.0f} | {last} | {behind} | {val:,.2f} |")
    for sid in SERIES:
        s = df[sid].dropna()
        spacing = s.index.to_series().diff().dt.days.median()
        last = s.index[-1].date()
        print(f"| {sid} | {SOURCE[sid]} | {len(s)} | {spacing:.0f} | {last} | {(RETRIEVED - last).days} | {s.iloc[-1]:,.2f} |")


if __name__ == "__main__":
    main()