"""fred-series-python-no-api-key.py — pull any FRED series as CSV without an API key, merge on date, and print a checked table.

What it does: downloads https://fred.stlouisfed.org/graph/fredgraph.csv?id=<ID> for each series id given on the
              command line (the same CSV the 'Download' button on a series page serves), turns FRED's "."
              missing marker into an empty cell, merges the series on date into one wide CSV, and prints the
              last rows plus a coverage check (first date, last date, missing count) so a stale or
              discontinued series is noticed rather than silently forward-filled.
Inputs      : series ids as arguments; --out path; --since YYYY-MM-DD. Example:
              python code/fred-series-python-no-api-key.py DGS10 DGS2 T10Y2Y --out datasets/fred-series-python-no-api-key.csv --since 2000-01-01
Outputs     : the CSV and a Markdown coverage table. Cite each series as https://fred.stlouisfed.org/series/<ID>
              with the retrieval date, because FRED revises many series.
Needs       : Python 3.13 standard library only (csv, urllib). pandas 3.0.2 is used only for the coverage table.
"""
import argparse
import csv
import datetime as dt
import io
import urllib.request

import pandas as pd

BASE = "https://fred.stlouisfed.org/graph/fredgraph.csv?id="


def fetch(series_id: str) -> dict[str, str]:
    req = urllib.request.Request(BASE + series_id, headers={"User-Agent": "prism-data-lab/1.0 (research)"})
    with urllib.request.urlopen(req, timeout=60) as r:
        text = r.read().decode("utf-8")
    rows = list(csv.reader(io.StringIO(text)))
    header = rows[0]
    if len(header) < 2:
        raise SystemExit(f"{series_id}: unexpected header {header!r} (bad id?)")
    return {row[0]: ("" if row[1] == "." else row[1]) for row in rows[1:] if row}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("series", nargs="+")
    ap.add_argument("--out", required=True)
    ap.add_argument("--since")
    a = ap.parse_args()
    data = {s: fetch(s) for s in a.series}
    dates = sorted(set().union(*[set(d) for d in data.values()]))
    if a.since:
        dates = [d for d in dates if d >= a.since]
    with open(a.out, "w", encoding="utf-8", newline="") as f:
        w = csv.writer(f)
        w.writerow(["date"] + a.series)
        for d in dates:
            w.writerow([d] + [data[s].get(d, "") for s in a.series])
    retrieved = dt.datetime.now(dt.timezone.utc).date().isoformat()
    print(f"wrote {len(dates)} rows x {len(a.series)} series to {a.out}; retrieved {retrieved}")
    df = pd.read_csv(a.out, parse_dates=["date"], index_col="date")
    print("| series | first date | last date | observations | empty cells in merged file |")
    print("|---|---|---|---|---|")
    for s in a.series:
        col = df[s]
        print(f"| {s} | {col.first_valid_index():%Y-%m-%d} | {col.last_valid_index():%Y-%m-%d} | {col.count()} | {int(col.isna().sum())} |")
    print("\nlast five rows:")
    print(df.tail(5).to_string())


if __name__ == "__main__":
    main()