bench: add conv-cache mode to stress bench

openai-proxy/src/bin/stress.rs

@@ -310,7 +310,7 @@ async fn main() -> anyhow::Result<()> {
     println!();
 
     // Hot mode: repeated payloads from fixed pool (L1 DashMap hits after warmup)
-    // First run warm up L1, second run is the measured one
+    // First run primes the message-content L1; second run is measured.
     run_sweep(&proxy_url, 32, base_iters, Some(hot_pool.clone())).await; // prime L1
     for &conc in conc_levels {
         let (el, mib, rps, p50, p95, p99, p999, errs) =
@@ -319,24 +319,53 @@ async fn main() -> anyhow::Result<()> {
         results.push(("hot".into(), conc, el, mib, rps, p50, p95, p99, p999, errs));
     }
 
+    println!();
+
+    // Conv-cache mode: same pool, but now conv_l1 is warm (full response cached).
+    // Every request should short-circuit at get_conv() — zero upstream calls.
+    let mock_hits_before = counters.hits.load(Ordering::Relaxed);
+    run_sweep(&proxy_url, 32, base_iters, Some(hot_pool.clone())).await; // prime conv_l1
+    let conv_hits_baseline = counters.hits.load(Ordering::Relaxed) - mock_hits_before;
+    for &conc in conc_levels {
+        let mock_before = counters.hits.load(Ordering::Relaxed);
+        let (el, mib, rps, p50, p95, p99, p999, errs) =
+            run_sweep(&proxy_url, conc, base_iters, Some(hot_pool.clone())).await;
+        let mock_after = counters.hits.load(Ordering::Relaxed);
+        let upstream_calls = mock_after - mock_before;
+        print_row("conv", conc, el, mib, rps, p50, p95, p99, p999, errs);
+        // Print upstream call count inline as a sanity check
+        if upstream_calls > 0 {
+            eprintln!("  [conv conc={conc}] upstream calls: {upstream_calls} (expected ~0 for conv cache hits)");
+        }
+        results.push(("conv".into(), conc, el, mib, rps, p50, p95, p99, p999, errs));
+    }
+    let _ = conv_hits_baseline; // used only to prime
+
     // ── 6. Summary ────────────────────────────────────────────────────────────
     let peak_cold = results.iter().filter(|r| r.0 == "cold")
         .max_by(|a, b| a.4.partial_cmp(&b.4).unwrap()).unwrap();
     let peak_hot  = results.iter().filter(|r| r.0 == "hot")
         .max_by(|a, b| a.4.partial_cmp(&b.4).unwrap()).unwrap();
+    let peak_conv = results.iter().filter(|r| r.0 == "conv")
+        .max_by(|a, b| a.4.partial_cmp(&b.4).unwrap()).unwrap();
 
     let cold_base = results.iter().find(|r| r.0 == "cold" && r.1 == 1).map(|r| r.4).unwrap_or(1.0);
     let hot_base  = results.iter().find(|r| r.0 == "hot"  && r.1 == 1).map(|r| r.4).unwrap_or(1.0);
+    let conv_base = results.iter().find(|r| r.0 == "conv" && r.1 == 1).map(|r| r.4).unwrap_or(1.0);
 
     eprintln!("\n╔══════════════════════════════════════════════════════════╗");
     eprintln!("  Peak COLD (unique payloads, full proxy path)");
     eprintln!("    {:>9.1} req/s  {:>7.1} MiB/s  conc={}", peak_cold.4, peak_cold.3, peak_cold.1);
     eprintln!("    p50={:.2}ms  p95={:.2}ms  p99={:.2}ms", peak_cold.5, peak_cold.6, peak_cold.7);
     eprintln!();
-    eprintln!("  Peak HOT  (L1 DRAM cache hits)");
+    eprintln!("  Peak HOT  (msg-content L1 DRAM hits)");
     eprintln!("    {:>9.1} req/s  {:>7.1} MiB/s  conc={}", peak_hot.4, peak_hot.3, peak_hot.1);
     eprintln!("    p50={:.2}ms  p95={:.2}ms  p99={:.2}ms", peak_hot.5, peak_hot.6, peak_hot.7);
     eprintln!();
+    eprintln!("  Peak CONV (conv-id L1 full-response cache hits)");
+    eprintln!("    {:>9.1} req/s  {:>7.1} MiB/s  conc={}", peak_conv.4, peak_conv.3, peak_conv.1);
+    eprintln!("    p50={:.2}ms  p95={:.2}ms  p99={:.2}ms", peak_conv.5, peak_conv.6, peak_conv.7);
+    eprintln!();
     eprintln!("  Scaling (cold, relative to conc=1 = {:.1} req/s)", cold_base);
     for r in results.iter().filter(|r| r.0 == "cold") {
         let eff = (r.4 / cold_base) / r.1 as f64 * 100.0;
@@ -348,6 +377,12 @@ async fn main() -> anyhow::Result<()> {
         let eff = (r.4 / hot_base) / r.1 as f64 * 100.0;
         eprintln!("    conc={:>3}  {:>9.1} req/s  {:.2}x  eff={:.1}%", r.1, r.4, r.4/hot_base, eff);
     }
+    eprintln!();
+    eprintln!("  Scaling (conv, relative to conc=1 = {:.1} req/s)", conv_base);
+    for r in results.iter().filter(|r| r.0 == "conv") {
+        let eff = (r.4 / conv_base) / r.1 as f64 * 100.0;
+        eprintln!("    conc={:>3}  {:>9.1} req/s  {:.2}x  eff={:.1}%", r.1, r.4, r.4/conv_base, eff);
+    }
     eprintln!("╚══════════════════════════════════════════════════════════╝");
 
     // Check 10k target