Understanding Results

{ "command": "test", "data": { }, "errors": [], "ok": true,
  "schema": "novetest/v1", "warnings": [] }

if   code == 0: handle_success(env)        # ok: true
elif code == 3: handle_test_failures(env)  # ok: true — read recommendations
elif code == 2: handle_usage_error(env)    # ok: false — fix the call
elif code == 4: handle_engine_missing(env) # ok: false — install/configure engine
elif code == 5: handle_storage_error(env)  # ok: false — store damaged
elif code == 1: handle_generic_error(env)  # ok: false — report as bug

recs = sorted(env["data"]["recommendations"], key=lambda r: r["priority"])
if not recs:
    return
top = recs[0]
cat = top["category"]

if cat == "all_green":
    return                                  # nothing to do
elif cat in {"investigate_location", "regression_with_localization"}:
    # rank-bearing slots: file, primary_line, line_range, rank, symbol, formula, mode.
    # Up to and including v0.2.1 this array is NOT score-ordered; from v0.3.0
    # it IS: rank (asc) then score_normalized (desc). Ties still share a
    # priority either way — select on the fields below, not on position:
    key  = lambda r: (r["slots"]["rank"], -r["slots"]["score_normalized"])
    locs = [r for r in recs if r["category"] == cat]
    best = min(locs, key=key)
    # A tie on BOTH rank and score means the analysis could not separate those
    # locations; what is left of the order is decided by file path. Every tied
    # entry is a co-equal suspect — carry them all, not just the first.
    tied = [r for r in locs if key(r) == key(best)]
    targets = [(r["slots"]["file"], r["slots"]["primary_line"]) for r in tied]
elif cat == "coverage_gap":
    # NOT rank-bearing: slots are file, lines (a list), mode, and
    # related_finding_id ("entry_index_<i>" into the finding's entries) —
    # no rank, primary_line, symbol or score_normalized. No in-category
    # selector exists; treat every coverage_gap entry as a peer.
    gaps = [(r["slots"]["file"], r["slots"]["lines"])
            for r in recs if r["category"] == "coverage_gap"]
elif cat == "investigate_regression":
    test_id = top["slots"]["test_id"]       # newly-failing test; no rank slots
elif cat == "unavailable_analysis":
    stages  = top["slots"]["unavailable_stages"]
    reasons = top["slots"]["reason_per_stage"]   # informational

{
  "command": "test",
  "data": {
    "recommendations": [
      { "category": "regression_with_localization", "priority": 1,
        "summary": "Test `tests/test_arithmetic.py::test_subtract` newly failing in this run; suspected location `subtract`@6 in `calc/arithmetic.py` (rank 1, ochiai=1.000, sbfl_per_test).",
        "slots": { "file": "calc/arithmetic.py", "formula": "ochiai",
                   "line_range": [5, 6], "mode": "sbfl_per_test", "primary_line": 6,
                   "rank": 1, "regression_kind": "newly_failing",
                   "run_reference_from": "01KXMCG7BTVNR0HKJAC33J23BB",
                   "run_reference_to": "01KXMCGQC43CBKJG5KQXTPW1S6",
                   "score_normalized": 1.0, "symbol": "subtract",
                   "test_id": "tests/test_arithmetic.py::test_subtract" } },
      { "category": "regression_with_localization", "priority": 1,
        "summary": "Test `tests/test_arithmetic.py::test_subtract` newly failing in this run; suspected location `test_subtract`@13 in `tests/test_arithmetic.py` (rank 1, ochiai=1.000, sbfl_per_test).",
        "slots": { "file": "tests/test_arithmetic.py", "formula": "ochiai",
                   "line_range": [12, 13], "mode": "sbfl_per_test", "primary_line": 13,
                   "rank": 1, "regression_kind": "newly_failing",
                   "run_reference_from": "01KXMCG7BTVNR0HKJAC33J23BB",
                   "run_reference_to": "01KXMCGQC43CBKJG5KQXTPW1S6",
                   "score_normalized": 1.0, "symbol": "test_subtract",
                   "test_id": "tests/test_arithmetic.py::test_subtract" } }
    ],
    "run_reference": { "run_id": "01KXMCGQC43CBKJG5KQXTPW1S6", "...": "..." },
    "stage_eligibility": { "coverage": "available", "localization": "sbfl_per_test",
                           "regression": "available", "replay": "not_run" }
  },
  "errors": [], "ok": true, "schema": "novetest/v1", "warnings": []
}

"coverage_outcome": { "kind": "fact-set", "mapping_granularity": "per-test",
  "summary": { "percent_covered": 100.0, "num_statements": 11, "covered_statements": 11, "...": "..." } },
"regression_outcome": { "kind": "fact-set",
  "summary": { "regressed": 1, "still_passing": 2, "total_target_tests": 3, "...": "..." }, "...": "..." },
"localization_outcome": { "kind": "fact-set", "mode": "sbfl_per_test", "confidence": "high",
  "formula": "ochiai", "top_n": 10, "entries": [ "...2 ranked entries..." ] },
"replay_outcome": { "kind": "unavailable", "reason": "missing-derived-facts",
  "detail": "no replay attempt has been made for this run" }

"summary": { "regressed": 1, "fixed": 0, "still_passing": 2, "still_failing": 0,
  "total_baseline_tests": 3, "total_target_tests": 3, "added": 0, "removed": 0,
  "newly_active": 0, "newly_skipped": 0, "still_skipped": 0 }

"replay_outcome": {
  "classification": "reproducible",
  "consistency_summary": { "original_passed": 0, "original_failed": 1,
    "replay_passed": 0, "replay_failed": 1, "replay_errored": 0 },
  "per_rerun_outcomes": ["failed"], "reruns_total": 1, "reruns_failed": 0,
  "replayed_run_reference": { "run_id": "01KVYRRVYB6K156ABEDFQTMQCG", "...": "..." },
  "test_id": null, "reason": null, "attempted_at": 1782370299982, "kind": "replay-result"
}