nampdn/openai-proxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bench: add conv-cache mode to stress bench

Browse Source
This commit is contained in:
nampdn
2026-06-08 15:11:54 +07:00
parent 1d0cf515ed
commit 7ab1fab51b
1 changed files with 37 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openai-proxy/src/bin/stress.rs
+37 -2
View File
@@ -310,7 +310,7 @@ async fn main() -> anyhow::Result<()> {
println!();
// Hot mode: repeated payloads from fixed pool (L1 DashMap hits after warmup)
// First run warm up L1, second run is the measured one
// First run primes the message-content L1; second run is measured.
run_sweep(&proxy_url, 32, base_iters, Some(hot_pool.clone())).await; // prime L1
for &conc in conc_levels {
let (el, mib, rps, p50, p95, p99, p999, errs) =
@@ -319,24 +319,53 @@ async fn main() -> anyhow::Result<()> {
results.push(("hot".into(), conc, el, mib, rps, p50, p95, p99, p999, errs));
}
println!();
// Conv-cache mode: same pool, but now conv_l1 is warm (full response cached).
// Every request should short-circuit at get_conv() — zero upstream calls.
let mock_hits_before = counters.hits.load(Ordering::Relaxed);
run_sweep(&proxy_url, 32, base_iters, Some(hot_pool.clone())).await; // prime conv_l1
let conv_hits_baseline = counters.hits.load(Ordering::Relaxed) - mock_hits_before;
for &conc in conc_levels {
let mock_before = counters.hits.load(Ordering::Relaxed);
let (el, mib, rps, p50, p95, p99, p999, errs) =
run_sweep(&proxy_url, conc, base_iters, Some(hot_pool.clone())).await;
let mock_after = counters.hits.load(Ordering::Relaxed);
let upstream_calls = mock_after - mock_before;
print_row("conv", conc, el, mib, rps, p50, p95, p99, p999, errs);
// Print upstream call count inline as a sanity check
if upstream_calls > 0 {
eprintln!(" [conv conc={conc}] upstream calls: {upstream_calls} (expected ~0 for conv cache hits)");
}
results.push(("conv".into(), conc, el, mib, rps, p50, p95, p99, p999, errs));
}
let _ = conv_hits_baseline; // used only to prime
// ── 6. Summary ────────────────────────────────────────────────────────────
let peak_cold = results.iter().filter(|r| r.0 == "cold")
.max_by(|a, b| a.4.partial_cmp(&b.4).unwrap()).unwrap();
let peak_hot = results.iter().filter(|r| r.0 == "hot")
.max_by(|a, b| a.4.partial_cmp(&b.4).unwrap()).unwrap();
let peak_conv = results.iter().filter(|r| r.0 == "conv")
.max_by(|a, b| a.4.partial_cmp(&b.4).unwrap()).unwrap();
let cold_base = results.iter().find(|r| r.0 == "cold" && r.1 == 1).map(|r| r.4).unwrap_or(1.0);
let hot_base = results.iter().find(|r| r.0 == "hot" && r.1 == 1).map(|r| r.4).unwrap_or(1.0);
let conv_base = results.iter().find(|r| r.0 == "conv" && r.1 == 1).map(|r| r.4).unwrap_or(1.0);
eprintln!("\n╔══════════════════════════════════════════════════════════╗");
eprintln!(" Peak COLD (unique payloads, full proxy path)");
eprintln!(" {:>9.1} req/s {:>7.1} MiB/s conc={}", peak_cold.4, peak_cold.3, peak_cold.1);
eprintln!(" p50={:.2}ms p95={:.2}ms p99={:.2}ms", peak_cold.5, peak_cold.6, peak_cold.7);
eprintln!();
eprintln!(" Peak HOT (L1 DRAM cache hits)");
eprintln!(" Peak HOT (msg-content L1 DRAM hits)");
eprintln!(" {:>9.1} req/s {:>7.1} MiB/s conc={}", peak_hot.4, peak_hot.3, peak_hot.1);
eprintln!(" p50={:.2}ms p95={:.2}ms p99={:.2}ms", peak_hot.5, peak_hot.6, peak_hot.7);
eprintln!();
eprintln!(" Peak CONV (conv-id L1 full-response cache hits)");
eprintln!(" {:>9.1} req/s {:>7.1} MiB/s conc={}", peak_conv.4, peak_conv.3, peak_conv.1);
eprintln!(" p50={:.2}ms p95={:.2}ms p99={:.2}ms", peak_conv.5, peak_conv.6, peak_conv.7);
eprintln!();
eprintln!(" Scaling (cold, relative to conc=1 = {:.1} req/s)", cold_base);
for r in results.iter().filter(|r| r.0 == "cold") {
let eff = (r.4 / cold_base) / r.1 as f64 * 100.0;
@@ -348,6 +377,12 @@ async fn main() -> anyhow::Result<()> {
let eff = (r.4 / hot_base) / r.1 as f64 * 100.0;
eprintln!(" conc={:>3} {:>9.1} req/s {:.2}x eff={:.1}%", r.1, r.4, r.4/hot_base, eff);
}
eprintln!();
eprintln!(" Scaling (conv, relative to conc=1 = {:.1} req/s)", conv_base);
for r in results.iter().filter(|r| r.0 == "conv") {
let eff = (r.4 / conv_base) / r.1 as f64 * 100.0;
eprintln!(" conc={:>3} {:>9.1} req/s {:.2}x eff={:.1}%", r.1, r.4, r.4/conv_base, eff);
}
eprintln!("╚══════════════════════════════════════════════════════════╝");
// Check 10k target