#!/usr/bin/env python3
"""Scores a photo calorie reader on the Marble open benchmark. Python 3.8+, no dependencies.

    python3 score.py truth.csv predictions.csv            # prints the scorecard
    python3 score.py truth.csv predictions.csv --json     # the same, as JSON

truth.csv:        id,kcal[,cluster]      one row per photo (cluster: the participant or dish group, if any)
predictions.csv:  id,kcal[,cost_usd,seconds]

Rules the scorer enforces:
- Every truth id must have exactly one prediction. A missing, empty, negative or non-numeric prediction scores as
  0 kcal (a failed read counts as a miss, as it would in an app), and is counted in `failed`.
- Extra ids in predictions.csv are an error: the set is fixed.
- Accuracy: mean absolute error in kcal (MAE), median absolute percentage error (MdAPE), signed bias
  (mean of predicted - true), share within 25% of the truth, each over every truth row.
- A 95% interval on MAE by bootstrap over clusters (rows with no cluster are their own cluster), seed 20261019.
- Cost and seconds per photo, if given, are reported as means and medians; they are what the submitter measured.
"""
import argparse
import csv
import json
import random
import statistics as st
import sys


def read_csv(path, need):
    with open(path, newline="") as f:
        rows = list(csv.DictReader(f))
    if not rows or any(k not in rows[0] for k in need):
        sys.exit(f"{path}: needs columns {', '.join(need)}")
    return rows


def num(x):
    try:
        v = float(x)
    except (TypeError, ValueError):
        return None
    return v if v == v and v >= 0 else None  # NaN and negatives are not answers


def score(truth_rows, pred_rows, boots=2000, seed=20261019):
    truth = {}
    for r in truth_rows:
        if r["id"] in truth:
            sys.exit(f"truth: duplicate id {r['id']}")
        t = num(r["kcal"])
        if t is None:
            sys.exit(f"truth: bad kcal for {r['id']}")
        truth[r["id"]] = (t, r.get("cluster") or r["id"])
    preds = {}
    for r in pred_rows:
        if r["id"] in preds:
            sys.exit(f"predictions: duplicate id {r['id']}")
        if r["id"] not in truth:
            sys.exit(f"predictions: id {r['id']} is not in the set")
        preds[r["id"]] = r
    err, ape, failed, cost, secs, clusters = [], [], 0, [], [], {}
    for i, (t, c) in truth.items():
        p = num((preds.get(i) or {}).get("kcal"))
        if p is None:
            failed, p = failed + 1, 0.0
        e = p - t
        err.append(e)
        if t > 0:
            ape.append(abs(e) / t * 100)
        clusters.setdefault(c, []).append(abs(e))
        for key, out in (("cost_usd", cost), ("seconds", secs)):
            v = num((preds.get(i) or {}).get(key))
            if v is not None:
                out.append(v)
    keys = list(clusters)
    rng = random.Random(seed)
    bs = []
    for _ in range(boots):
        s = [clusters[keys[rng.randrange(len(keys))]] for _ in keys]
        bs.append(sum(map(sum, s)) / sum(map(len, s)))
    bs.sort()
    out = {"n": len(truth), "failed": failed, "mae_kcal": round(st.mean(abs(e) for e in err), 1),
           "mae_95": [round(bs[int(0.025 * boots)], 1), round(bs[int(0.975 * boots)], 1)],
           "mdape_pct": round(st.median(ape), 1), "bias_kcal": round(st.mean(err), 1),
           "within_25pct": round(100 * sum(a <= 25 for a in ape) / len(ape), 1)}
    if cost:
        out.update(cost_usd_mean=round(st.mean(cost), 5), cost_reported=len(cost))
    if secs:
        out.update(seconds_mean=round(st.mean(secs), 2), seconds_median=round(st.median(secs), 2), seconds_reported=len(secs))
    return out


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("truth")
    ap.add_argument("predictions")
    ap.add_argument("--json", action="store_true")
    a = ap.parse_args()
    s = score(read_csv(a.truth, ["id", "kcal"]), read_csv(a.predictions, ["id", "kcal"]))
    if a.json:
        print(json.dumps(s, indent=1))
        return
    print(f"photos {s['n']}  failed {s['failed']}")
    print(f"MAE {s['mae_kcal']} kcal (95% {s['mae_95'][0]}-{s['mae_95'][1]})  MdAPE {s['mdape_pct']}%  bias {s['bias_kcal']:+} kcal  within 25%: {s['within_25pct']}%")
    if "cost_usd_mean" in s:
        print(f"cost ${s['cost_usd_mean']} per photo ({s['cost_reported']} reported)")
    if "seconds_mean" in s:
        print(f"seconds {s['seconds_median']} median, {s['seconds_mean']} mean ({s['seconds_reported']} reported)")


if __name__ == "__main__":
    main()