{
 "generated_at": "2026-10-14 14:09:54",
 "pipeline": {
  "stt": "saaras:v3 -> whisper-large-v3",
  "llm": "openai/gpt-oss-120b",
  "tts": "bulbul:v3",
  "script_generator": "qwen/qwen3.8-27b",
  "thresholds": {
   "medium": 40,
   "high": 68
  }
 },
 "positive_class": "fraud; predicted fraud = HIGH alert fired at any point in the call",
 "caveat": "Synthetic, LLM-written, TTS-voiced calls. Treat as an optimistic upper bound.",
 "set": {
  "n": 46,
  "fraud": 24,
  "genuine": 22
 },
 "conditions": {
  "clean": {
   "metrics": {
    "n": 46,
    "llm_steps": 142,
    "llm_failed_steps": 42,
    "llm_failure_rate": 0.2958,
    "n_fraud": 24,
    "n_genuine": 22,
    "n_errors": 0,
    "accuracy": 0.8913,
    "precision": 0.913,
    "recall": 0.875,
    "f1": 0.8936,
    "false_alarm_rate": 0.0909,
    "confusion": {
     "tp": 21,
     "fp": 2,
     "tn": 20,
     "fn": 3
    },
    "medium_or_above": {
     "recall": 1.0,
     "false_alarm_rate": 0.3182
    },
    "median_time_to_alert_s": 60.08,
    "time_to_alert_n": 21,
    "median_alert_lag_s": 32.47,
    "p95_alert_lag_s": 60.09,
    "early_warning_rate": 0.6667,
    "scenario_accuracy": 0.6667,
    "per_language": {
     "en": {
      "n": 14,
      "accuracy": 0.7143,
      "false_alarm_rate": 0.1667
     },
     "hi": {
      "n": 15,
      "accuracy": 1.0,
      "false_alarm_rate": 0.0
     },
     "hinglish": {
      "n": 17,
      "accuracy": 0.9412,
      "false_alarm_rate": 0.1111
     }
    },
    "per_scenario": {
     "courier_scam": {
      "n": 3,
      "detected": 3,
      "recall": 1.0
     },
     "digital_arrest": {
      "n": 7,
      "detected": 7,
      "recall": 1.0
     },
     "fake_bank_kyc": {
      "n": 7,
      "detected": 6,
      "recall": 0.8571
     },
     "otp_urgent_payment": {
      "n": 7,
      "detected": 5,
      "recall": 0.7143
     }
    },
    "per_genuine_kind": {
     "bank_alert_readout": {
      "n": 4,
      "false_alarms": 1
     },
     "agent_warns_no_otp": {
      "n": 4,
      "false_alarms": 1
     },
     "delivery_address": {
      "n": 4,
      "false_alarms": 0
     },
     "relative_money": {
      "n": 4,
      "false_alarms": 0
     },
     "benign_other": {
      "n": 6,
      "false_alarms": 0
     }
    },
    "failures": 5
   },
   "subset": false,
   "failures": [
    {
     "id": "fake_bank_kyc_en_02",
     "true_label": "fraud",
     "predicted_level": "medium",
     "score": 52.2,
     "language": "en",
     "why": "MISSED: peaked at medium 52.2 (HIGH needs 68). The LLM was right (88.0) but the rule layer found only 12.0 (Claims to be your bank), and the weighted average pulled the fused score below HIGH. Fusion flaw: absent rule hits are treated as evidence of innocence. STT: sarvam:saaras:v3."
    },
    {
     "id": "otp_urgent_payment_en_03",
     "true_label": "fraud",
     "predicted_level": "medium",
     "score": 62.8,
     "language": "en",
     "why": "MISSED: peaked at medium 62.8 (HIGH needs 68). The LLM was right (92.0) but the rule layer found only 30.0 (Asks for OTP), and the weighted average pulled the fused score below HIGH. Fusion flaw: absent rule hits are treated as evidence of innocence. STT: sarvam:saaras:v3."
    },
    {
     "id": "otp_urgent_payment_en_02",
     "true_label": "fraud",
     "predicted_level": "medium",
     "score": 53.4,
     "language": "en",
     "why": "MISSED: peaked at medium 53.4 (HIGH needs 68). The LLM was right (92.0) but the rule layer found only 10.0 (Impersonates TRAI / telecom), and the weighted average pulled the fused score below HIGH. Fusion flaw: absent rule hits are treated as evidence of innocence. STT: sarvam:saaras:v3."
    },
    {
     "id": "bank_alert_readout_en_02",
     "true_label": "genuine",
     "predicted_level": "high",
     "score": 92.0,
     "language": "en",
     "why": "FALSE ALARM: reached high 92.0. The LLM call failed (429 rate limited; waited 109.0s) on 2 step(s), so the rules-only fallback decided. Rules fired on: Artificial urgency; Asks for OTP; Claims to be your bank; Explicitly asks you to share a secret. No genuine-call marker matched the caller's wording. STT: sarvam:saaras:v3."
    },
    {
     "id": "agent_warns_no_otp_hinglish_01",
     "true_label": "genuine",
     "predicted_level": "high",
     "score": 68.0,
     "language": "hinglish",
     "why": "FALSE ALARM: reached high 68.0. The LLM call failed (429 rate limited; waited 143.0s) on 4 step(s), so the rules-only fallback decided. Rules fired on: Asks for OTP; Claims to be your bank; Impersonates police / CBI / agency. No genuine-call marker matched the caller's wording. STT: sarvam:saaras:v3."
    }
   ],
   "errors": []
  },
  "snr10": {
   "metrics": {
    "n": 24,
    "llm_steps": 24,
    "llm_failed_steps": 19,
    "llm_failure_rate": 0.7917,
    "n_fraud": 12,
    "n_genuine": 12,
    "n_errors": 0,
    "accuracy": 0.875,
    "precision": 1.0,
    "recall": 0.75,
    "f1": 0.8571,
    "false_alarm_rate": 0.0,
    "confusion": {
     "tp": 9,
     "fp": 0,
     "tn": 12,
     "fn": 3
    },
    "medium_or_above": {
     "recall": 1.0,
     "false_alarm_rate": 0.1667
    },
    "median_time_to_alert_s": 61.0,
    "time_to_alert_n": 9,
    "median_alert_lag_s": 26.15,
    "p95_alert_lag_s": 35.47,
    "early_warning_rate": 0.0,
    "scenario_accuracy": 0.1667,
    "per_language": {
     "en": {
      "n": 7,
      "accuracy": 0.7143,
      "false_alarm_rate": 0.0
     },
     "hi": {
      "n": 10,
      "accuracy": 0.9,
      "false_alarm_rate": 0.0
     },
     "hinglish": {
      "n": 7,
      "accuracy": 1.0,
      "false_alarm_rate": 0.0
     }
    },
    "per_scenario": {
     "courier_scam": {
      "n": 3,
      "detected": 3,
      "recall": 1.0
     },
     "digital_arrest": {
      "n": 2,
      "detected": 2,
      "recall": 1.0
     },
     "fake_bank_kyc": {
      "n": 2,
      "detected": 2,
      "recall": 1.0
     },
     "otp_urgent_payment": {
      "n": 5,
      "detected": 2,
      "recall": 0.4
     }
    },
    "per_genuine_kind": {
     "agent_warns_no_otp": {
      "n": 2,
      "false_alarms": 0
     },
     "delivery_address": {
      "n": 2,
      "false_alarms": 0
     },
     "benign_other": {
      "n": 5,
      "false_alarms": 0
     },
     "relative_money": {
      "n": 3,
      "false_alarms": 0
     }
    },
    "failures": 3
   },
   "subset": true,
   "failures": [
    {
     "id": "otp_urgent_payment_en_03",
     "true_label": "fraud",
     "predicted_level": "medium",
     "score": 62.8,
     "language": "en",
     "why": "MISSED: peaked at medium 62.8 (HIGH needs 68). The LLM was right (92.0) but the rule layer found only 30.0 (Asks for OTP), and the weighted average pulled the fused score below HIGH. Fusion flaw: absent rule hits are treated as evidence of innocence. STT: sarvam:saaras:v3."
    },
    {
     "id": "otp_urgent_payment_hi_02",
     "true_label": "fraud",
     "predicted_level": "medium",
     "score": 42.0,
     "language": "hi",
     "why": "MISSED: peaked at medium 42.0 (HIGH needs 68). LLM unavailable (429 rate limited; waited 65.0s); rules alone scored 34.0. STT: sarvam:saaras:v3."
    },
    {
     "id": "otp_urgent_payment_en_01",
     "true_label": "fraud",
     "predicted_level": "medium",
     "score": 50.0,
     "language": "en",
     "why": "MISSED: peaked at medium 50.0 (HIGH needs 68). LLM unavailable (429 rate limited; waited 241.0s); rules alone scored 42.0. STT: sarvam:saaras:v3."
    }
   ],
   "errors": []
  }
 }
}