{
  "meta": {
    "title": "Decision Models evidence: serving speed and quality, raw numbers",
    "date_measured": "2026-10-11",
    "published_note": "All serving measurements below are self-measured by Decision Models on 11 October 2026. None has been audited by a third party.",
    "hardware": "One NVIDIA RTX PRO 6000 Blackwell (96 GB class) per measurement session.",
    "units": {
      "di_dec_s": "decisions per second on Decision Index traffic (1,000 states, 5,129 decisions), best tested concurrency",
      "p300_dec_s": "valid decisions per second on the public JevBench P300 items, best tested concurrency",
      "ratio_*": "throughput divided by the reference arm in the same session; ratio_di_cross_session divides by 4.880 from a different session",
      "*_delta": "score points versus the reference arm on the same rows; *_ci_* is the paired bootstrap 95 % interval (1,000 resamples)",
      "p50_c1_ms": "median latency in milliseconds of the first 50 one-at-a-time requests (only measured for the same-pod Decider runs)",
      "c1_flips": "answers that changed between two identical one-at-a-time passes over c1_pairs items"
    },
    "quality_gate": "Every 95 % interval of the paired score difference versus the reference arm includes 0 or is positive, and the c1 repeat has 0 answer flips.",
    "decision_index_traffic": "A fixed sample of 1,000 states (5,129 decisions: every fifth of 5,000 fixed IDs) from public Decision Index tasks.",
    "p300": "The 300 public JevBench v1.6 P300 items (297 valid for Decider; three identical over-context failures in every arm)."
  },
  "leaderboards": {
    "decision_index": {
      "version": "0.3.1",
      "data_updated": "2026-10-10",
      "perplexity_decider_v1_1": 62.75,
      "jev_1_13_0": 60.11,
      "perplexity_rank": 1,
      "url": "https://huggingface.co/spaces/multimodalart/jev-decision-index",
      "note": "Third-party board. Full score. Perplexity Decider v1.1 is the open model, not the Decision Models service."
    },
    "jevbench": {
      "version": "1.6.1",
      "capability_perplexity_decider_v1_1": 82.79,
      "capability_jev_1_13_0": 77.13,
      "url": "https://benchmarkheaven.com/jev-models/v1.6.1",
      "note": "JevBench is run by the founder of Decision Models. Capability score = mean of Intelligence and Calibration; highest among systems inside the published median-latency cap; outside the cost cap, so not ranked first on the default view. Decision Models offerings are not listed on JevBench."
    }
  },
  "rows": [
    {
      "id": "pplx-author-bf16",
      "section": "s1-pro",
      "model": "Perplexity Decider v1.1 27B",
      "api_id": "s1-pro",
      "arm": "Author reference code, BF16",
      "session": "same-pod research run",
      "reference_arm": null,
      "di_dec_s": 4.88,
      "p300_dec_s": 8.465,
      "ratio_di": 1.0,
      "ratio_p300": 1.0,
      "ratio_di_cross_session": null,
      "di_delta": null,
      "di_ci_low": null,
      "di_ci_high": null,
      "p300_delta": null,
      "p300_ci_low": null,
      "p300_ci_high": null,
      "c1_flips": null,
      "c1_pairs": null,
      "p50_c1_ms": 115.548,
      "note": "Reference arm. Determinism not re-run."
    },
    {
      "id": "pplx-fp8-samepod",
      "section": "s1-pro",
      "model": "Perplexity Decider v1.1 27B",
      "api_id": "s1-pro",
      "arm": "FP8 build",
      "session": "same-pod research run",
      "reference_arm": "pplx-author-bf16",
      "di_dec_s": 14.291,
      "p300_dec_s": 19.355,
      "ratio_di": 2.929,
      "ratio_p300": 2.287,
      "ratio_di_cross_session": null,
      "di_delta": 0.9,
      "di_ci_low": -0.26,
      "di_ci_high": 2.29,
      "p300_delta": -1.23,
      "p300_ci_low": -1.91,
      "p300_ci_high": 0.34,
      "c1_flips": null,
      "c1_pairs": null,
      "p50_c1_ms": 85.206,
      "note": "Determinism not re-run."
    },
    {
      "id": "pplx-nvfp4-mid-samepod",
      "section": "s1-pro",
      "model": "Perplexity Decider v1.1 27B",
      "api_id": "s1-pro",
      "arm": "NVFP4, earlier build (middle layers only)",
      "session": "same-pod research run",
      "reference_arm": "pplx-author-bf16",
      "di_dec_s": 20.35,
      "p300_dec_s": 27.548,
      "ratio_di": 4.17,
      "ratio_p300": 3.254,
      "ratio_di_cross_session": null,
      "di_delta": 0.7,
      "di_ci_low": -1.04,
      "di_ci_high": 2.44,
      "p300_delta": 0.69,
      "p300_ci_low": -1.28,
      "p300_ci_high": 1.96,
      "c1_flips": null,
      "c1_pairs": null,
      "p50_c1_ms": 84.324,
      "note": "Determinism not re-run."
    },
    {
      "id": "pplx-nvfp4-expanded-samepod",
      "section": "s1-pro",
      "model": "Perplexity Decider v1.1 27B",
      "api_id": "s1-pro",
      "arm": "NVFP4, expanded build",
      "session": "same-pod research run",
      "reference_arm": "pplx-author-bf16",
      "di_dec_s": 21.825,
      "p300_dec_s": 29.794,
      "ratio_di": 4.473,
      "ratio_p300": 3.52,
      "ratio_di_cross_session": null,
      "di_delta": -0.34,
      "di_ci_low": -2.38,
      "di_ci_high": 1.63,
      "p300_delta": 0.45,
      "p300_ci_low": -1.78,
      "p300_ci_high": 1.93,
      "c1_flips": 0,
      "c1_pairs": 297,
      "p50_c1_ms": 86.855,
      "note": "Two full c1 passes over the public P300 items: 297 valid pairs, 0 answer flips."
    },
    {
      "id": "pplx-fp8-image",
      "section": "s1-pro",
      "model": "Perplexity Decider v1.1 27B",
      "api_id": "s1-pro",
      "arm": "FP8, production node image",
      "session": "production-image run",
      "reference_arm": "pplx-author-bf16 (not run in this session)",
      "di_dec_s": 14.544,
      "p300_dec_s": 20.464,
      "ratio_di": null,
      "ratio_p300": null,
      "ratio_di_cross_session": 2.98,
      "di_delta": -0.57,
      "di_ci_low": -1.64,
      "di_ci_high": 0.73,
      "p300_delta": -0.92,
      "p300_ci_low": -1.93,
      "p300_ci_high": 0.48,
      "c1_flips": null,
      "c1_pairs": null,
      "note": "Quality deltas are against the author reference scores from the same-pod run. Cross-session ratio is against 4.880 from a different session. P300: one concurrency-32 pass.",
      "p50_c1_ms": null
    },
    {
      "id": "pplx-nvfp4-mid-image",
      "section": "s1-pro",
      "model": "Perplexity Decider v1.1 27B",
      "api_id": "s1-pro",
      "arm": "NVFP4 (middle layers only), production node image",
      "session": "production-image run",
      "reference_arm": "pplx-author-bf16 (not run in this session)",
      "di_dec_s": 20.29,
      "p300_dec_s": 28.623,
      "ratio_di": null,
      "ratio_p300": null,
      "ratio_di_cross_session": 4.158,
      "di_delta": 0.26,
      "di_ci_low": -1.26,
      "di_ci_high": 1.97,
      "p300_delta": 0.15,
      "p300_ci_low": -1.82,
      "p300_ci_high": 1.58,
      "c1_flips": 0,
      "c1_pairs": null,
      "note": "0 flips at c1 on both the engine and the node. P300: one concurrency-32 pass.",
      "p50_c1_ms": null
    },
    {
      "id": "pplx-nvfp4-expanded-image",
      "section": "s1-pro",
      "model": "Perplexity Decider v1.1 27B",
      "api_id": "s1-pro",
      "arm": "NVFP4 expanded, production node image",
      "session": "production-image run",
      "reference_arm": "pplx-author-bf16 (not run in this session)",
      "di_dec_s": 21.644,
      "p300_dec_s": 29.316,
      "ratio_di": null,
      "ratio_p300": null,
      "ratio_di_cross_session": 4.435,
      "di_delta": -0.58,
      "di_ci_low": -2.5,
      "di_ci_high": 1.37,
      "p300_delta": 0.84,
      "p300_ci_low": -1.53,
      "p300_ci_high": 1.81,
      "c1_flips": 0,
      "c1_pairs": null,
      "note": "0 flips at c1 on both the engine and the node. P300: median of three concurrency-32 passes.",
      "p50_c1_ms": null
    },
    {
      "id": "blink-stock-vllm",
      "section": "blink",
      "model": "Blink v0.3 26B-A4B NVFP4",
      "api_id": "s1-fast, s1-vision",
      "arm": "Stock vLLM (not batch-invariant)",
      "session": "blink same-host run",
      "reference_arm": null,
      "di_dec_s": 138.19,
      "p300_dec_s": 53.49,
      "ratio_di": 1.0,
      "ratio_p300": 1.0,
      "ratio_di_cross_session": null,
      "di_delta": null,
      "di_ci_low": null,
      "di_ci_high": null,
      "p300_delta": null,
      "p300_ci_low": null,
      "p300_ci_high": null,
      "c1_flips": 6,
      "c1_pairs": 300,
      "note": "Reference arm. Fails the c1 determinism gate: 6 of 300 answers flipped between two passes.",
      "p50_c1_ms": null
    },
    {
      "id": "blink-batch-invariant-64",
      "section": "blink",
      "model": "Blink v0.3 26B-A4B NVFP4",
      "api_id": "s1-fast, s1-vision",
      "arm": "Batch-invariant, 64 sequences (before)",
      "session": "blink same-host run",
      "reference_arm": "blink-stock-vllm",
      "di_dec_s": 93.983,
      "p300_dec_s": 38.41,
      "ratio_di": null,
      "ratio_p300": null,
      "ratio_di_cross_session": null,
      "di_delta": -0.48,
      "di_ci_low": -2.19,
      "di_ci_high": 1.54,
      "p300_delta": 0.15,
      "p300_ci_low": -2.99,
      "p300_ci_high": 4.52,
      "c1_flips": 0,
      "c1_pairs": 300,
      "note": "DI: mean of two alternating fresh-server repeats; the first pass was 93.26. P300: first pass.",
      "p50_c1_ms": null
    },
    {
      "id": "blink-batch-invariant-256",
      "section": "blink",
      "model": "Blink v0.3 26B-A4B NVFP4",
      "api_id": "s1-fast, s1-vision",
      "arm": "Batch-invariant, 256 sequences, 16,384 batched tokens (after)",
      "session": "blink same-host run",
      "reference_arm": "blink-stock-vllm",
      "di_dec_s": 97.118,
      "p300_dec_s": 38.99,
      "ratio_di": null,
      "ratio_p300": null,
      "ratio_di_cross_session": null,
      "di_delta": -0.09,
      "di_ci_low": -1.88,
      "di_ci_high": 1.95,
      "p300_delta": 0.15,
      "p300_ci_low": -2.99,
      "p300_ci_high": 4.52,
      "c1_flips": 0,
      "c1_pairs": 300,
      "note": "DI: mean of two alternating fresh-server repeats; the first pass was 96.82 (70.1 % of the stock server). P300: first pass.",
      "p50_c1_ms": null
    },
    {
      "id": "quyet-stock-hf",
      "section": "quyet",
      "model": "Quyet-1.0-Large",
      "api_id": null,
      "arm": "Stock Hugging Face server (author reference)",
      "session": "quyet same-host run",
      "reference_arm": null,
      "di_dec_s": 5.5441,
      "p300_dec_s": 6.6267,
      "ratio_di": 1.0,
      "ratio_p300": 1.0,
      "ratio_di_cross_session": null,
      "di_delta": null,
      "di_ci_low": null,
      "di_ci_high": null,
      "p300_delta": null,
      "p300_ci_low": null,
      "p300_ci_high": null,
      "c1_flips": null,
      "c1_pairs": null,
      "note": "Reference arm. DI at client concurrency 4, P300 at concurrency 2. Determinism not re-run.",
      "p50_c1_ms": null
    },
    {
      "id": "quyet-fp8",
      "section": "quyet",
      "model": "Quyet-1.0-Large",
      "api_id": null,
      "arm": "vLLM, FP8",
      "session": "quyet same-host run",
      "reference_arm": "quyet-stock-hf",
      "di_dec_s": 24.8468,
      "p300_dec_s": 17.3419,
      "ratio_di": 4.482,
      "ratio_p300": 2.617,
      "ratio_di_cross_session": null,
      "di_delta": 0.4,
      "di_ci_low": -1.11,
      "di_ci_high": 1.84,
      "p300_delta": 1.019,
      "p300_ci_low": -0.9,
      "p300_ci_high": 2.945,
      "c1_flips": 0,
      "c1_pairs": 300,
      "note": "DI at concurrency 8, P300 at concurrency 32.",
      "p50_c1_ms": null
    },
    {
      "id": "quyet-nvfp4",
      "section": "quyet",
      "model": "Quyet-1.0-Large",
      "api_id": null,
      "arm": "vLLM, calibrated NVFP4 (all linear layers), BF16 KV cache",
      "session": "quyet same-host run",
      "reference_arm": "quyet-stock-hf",
      "di_dec_s": 36.7087,
      "p300_dec_s": 26.2594,
      "ratio_di": 6.621,
      "ratio_p300": 3.963,
      "ratio_di_cross_session": null,
      "di_delta": -0.12,
      "di_ci_low": -2.52,
      "di_ci_high": 1.66,
      "p300_delta": -1.376,
      "p300_ci_low": -4.075,
      "p300_ci_high": 1.321,
      "c1_flips": 0,
      "c1_pairs": 300,
      "note": "Both at concurrency 32. The DI Language sub-score is 3.34 points lower, 95 % CI -6.74 to -0.33 (excludes 0); aggregate gate passes.",
      "p50_c1_ms": null
    }
  ]
}