140 lines
4.3 KiB
Python
140 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Opt-in run-history logging for
|
|
docs/intent/board-intent-run-metrics-baseline.md.
|
|
|
|
A run is only ever recorded for a slug listed in the BOARD_METRICS_SLUGS
|
|
environment variable (a comma-separated allowlist) -- the intent's own
|
|
"kein automatisches, board-weites Tracking" scope limit, deliberately not
|
|
a per-card UI toggle (see the intent's "Umfang"). Every other card's
|
|
state.json entry is completely untouched by this module: record_run() is
|
|
a silent no-op for a slug that isn't enrolled. A run entry is pure
|
|
observation -- nothing here ever changes a card's phase or queues a job,
|
|
unlike every other module in this directory.
|
|
"""
|
|
|
|
import json
|
|
import state_store
|
|
import os
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
# Caps state.json growth for a long-running pilot slug -- see the intent's
|
|
# own note that a small, deliberately chosen pilot set, not unlimited
|
|
# history, is what's in scope.
|
|
MAX_RUN_HISTORY = 200
|
|
|
|
|
|
def _state_path():
|
|
return Path(os.environ["BOARD_STATE_FILE"])
|
|
|
|
|
|
def _load_state():
|
|
return state_store.load()
|
|
|
|
|
|
def _save_state(data):
|
|
state_store.save(data)
|
|
|
|
|
|
def now_iso():
|
|
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def enabled_slugs():
|
|
raw = os.environ.get("BOARD_METRICS_SLUGS", "")
|
|
return {slug.strip() for slug in raw.split(",") if slug.strip()}
|
|
|
|
|
|
def enabled(slug):
|
|
return slug in enabled_slugs()
|
|
|
|
|
|
def duration_since(requested_at_iso, *, now=None):
|
|
"""Seconds between an ISO timestamp in now_iso()'s exact format and
|
|
now (default: this moment). Returns None instead of raising on any
|
|
missing/malformed input -- a metrics field must never be able to crash
|
|
the caller's normal job-completion path."""
|
|
if not requested_at_iso:
|
|
return None
|
|
try:
|
|
started = datetime.strptime(str(requested_at_iso), "%Y-%m-%dT%H:%M:%SZ").replace(
|
|
tzinfo=timezone.utc
|
|
)
|
|
except ValueError:
|
|
return None
|
|
end = now or datetime.now(timezone.utc)
|
|
return max(0.0, (end - started).total_seconds())
|
|
|
|
|
|
@state_store.atomic
|
|
def record_run(
|
|
slug,
|
|
run_kind,
|
|
*,
|
|
status,
|
|
job_type=None,
|
|
runtime=None,
|
|
model=None,
|
|
reasoning_effort=None,
|
|
duration_seconds=None,
|
|
job_id=None,
|
|
commit_hash=None,
|
|
review_decision=None,
|
|
research_report=None,
|
|
report_path=None,
|
|
):
|
|
"""Append one immutable observation to card["run_history"]. run_kind is
|
|
"board" (went through the normal Builder/Reviewer/Research pipeline) or
|
|
"baseline_single_prompt" (metrics_baseline.py, no pipeline at all) --
|
|
see the intent's run_kind field. No-op, returns None, for any slug not
|
|
in BOARD_METRICS_SLUGS."""
|
|
if not enabled(slug):
|
|
return None
|
|
entry = {"recorded_at": now_iso(), "run_kind": run_kind, "status": status}
|
|
for key, value in (
|
|
("job_type", job_type),
|
|
("runtime", runtime),
|
|
("model", model),
|
|
("reasoning_effort", reasoning_effort),
|
|
("duration_seconds", duration_seconds),
|
|
("job_id", job_id),
|
|
("commit_hash", commit_hash),
|
|
("review_decision", review_decision),
|
|
("research_report", research_report),
|
|
("report_path", report_path),
|
|
):
|
|
if value is not None:
|
|
entry[key] = value
|
|
|
|
state = _load_state()
|
|
card = state["cards"].setdefault(slug, {})
|
|
history = card.setdefault("run_history", [])
|
|
history.append(entry)
|
|
if len(history) > MAX_RUN_HISTORY:
|
|
del history[: len(history) - MAX_RUN_HISTORY]
|
|
_save_state(state)
|
|
return entry
|
|
|
|
|
|
def summary(card):
|
|
"""Compact per-card rollup for the board UI: total runs, the board/
|
|
baseline split, and how many reached a successful outcome. None for a
|
|
card with no run_history (the vast majority -- unenrolled slugs)."""
|
|
history = card.get("run_history")
|
|
if not isinstance(history, list) or not history:
|
|
return None
|
|
total = len(history)
|
|
board_runs = sum(1 for entry in history if entry.get("run_kind") == "board")
|
|
baseline_runs = total - board_runs
|
|
successful = sum(
|
|
1
|
|
for entry in history
|
|
if entry.get("review_decision") == "bereit_fuer_deployment"
|
|
or entry.get("status") in ("done", "committed")
|
|
)
|
|
return {
|
|
"total": total,
|
|
"board": board_runs,
|
|
"baseline": baseline_runs,
|
|
"successful": successful,
|
|
}
|