{
  "source_commit": "fa3a0e27f49035816e2f7e59db95c5ae42d064af",
  "as_of": "2026-10-04",
  "corpora": [
    "scifact",
    "nfcorpus",
    "fiqa"
  ],
  "query_counts": [
    300,
    323,
    300
  ],
  "quality_metric": "nDCG@10; unweighted mean of three corpus means",
  "cost_basis": "Captured usage at run-time list prices; historical pricing conventions preserved; not invoices",
  "bm25": 0.4039205123793589,
  "rows": [
    {
      "key": "jev_batch_generic",
      "label": "Jev · batch",
      "family": "jev",
      "quality": 0.5008547536859029,
      "per_corpus": {
        "scifact": 0.7683290081756676,
        "nfcorpus": 0.35820059943282617,
        "fiqa": 0.37603465344921505
      },
      "usd_per_1k": 0.5372456140350877,
      "p50_ms": 223.14528624216715,
      "p95_ms": 1419.498046239217,
      "latency_unit": "query call",
      "latency_basis": "Historical runs; laptop, September"
    },
    {
      "key": "jev_pair_generic",
      "label": "Jev · pair",
      "family": "jev",
      "quality": 0.5021354827688203,
      "per_corpus": {
        "scifact": 0.7721709227158036,
        "nfcorpus": 0.35799232186049085,
        "fiqa": 0.3762432037301664
      },
      "usd_per_1k": 0.8670567595459235,
      "p50_ms": 131.45633538564047,
      "p95_ms": 255.9018929799398,
      "latency_unit": "document call",
      "latency_basis": "Historical runs; laptop, September"
    },
    {
      "key": "mixedbread_v31",
      "label": "Mixedbread v3.1 listwise",
      "family": "historical",
      "quality": 0.510609841223261,
      "per_corpus": {
        "scifact": 0.7674914198991412,
        "nfcorpus": 0.3666336737446514,
        "fiqa": 0.39770443002599043
      },
      "usd_per_1k": 1.41925,
      "p50_ms": 216.68759981791177,
      "p95_ms": 262.3496850331624,
      "latency_unit": "query call",
      "latency_basis": "Historical runs; laptop, September"
    },
    {
      "key": "voyage3",
      "label": "Voyage rerank-3",
      "family": "historical",
      "quality": 0.504438416759762,
      "per_corpus": {
        "scifact": 0.7546579707306361,
        "nfcorpus": 0.35669798964188537,
        "fiqa": 0.40195928990676455
      },
      "usd_per_1k": 0.5,
      "p50_ms": 185.2254867553711,
      "p95_ms": 286.7468198140462,
      "latency_unit": "query call",
      "latency_basis": "Historical runs; laptop, September"
    },
    {
      "key": "cohere",
      "label": "Cohere v3.5",
      "family": "historical",
      "quality": 0.48614827536480154,
      "per_corpus": {
        "scifact": 0.7454041499445426,
        "nfcorpus": 0.3394673420129891,
        "fiqa": 0.37357333413687294
      },
      "usd_per_1k": 2,
      "p50_ms": 191.98481241861978,
      "p95_ms": 456.168254216512,
      "latency_unit": "query call",
      "latency_basis": "Historical runs; laptop, September"
    },
    {
      "key": "mixedbread",
      "label": "Mixedbread large-v2",
      "family": "historical",
      "quality": 0.4755640209697933,
      "per_corpus": {
        "scifact": 0.7491112976585692,
        "nfcorpus": 0.32382252788004123,
        "fiqa": 0.3537582373707694
      },
      "usd_per_1k": 1.41925,
      "p50_ms": 360.6390953063965,
      "p95_ms": 428.5889466603597,
      "latency_unit": "query call",
      "latency_basis": "Historical runs; laptop, September"
    },
    {
      "key": "jina35",
      "label": "Jina v3.5*",
      "family": "historical",
      "quality": 0.4917333333333333,
      "per_corpus": {
        "scifact": 0.7505,
        "nfcorpus": 0.3449,
        "fiqa": 0.3798
      },
      "usd_per_1k": 0.47308333333333336,
      "p50_ms": 302.74200439453125,
      "p95_ms": 606.8589687347412,
      "latency_unit": "query call",
      "latency_basis": "Historical API SciFact only; quality from open weights"
    },
    {
      "key": "clef_flash_pair_generic",
      "label": "Clef-flash · pair",
      "family": "clef",
      "quality": 0.49832834925328084,
      "per_corpus": {
        "scifact": 0.7686173593041452,
        "nfcorpus": 0.35483538031282263,
        "fiqa": 0.3715323081428747
      },
      "usd_per_1k": 1.3854159504643964,
      "p50_ms": 262.52713869325817,
      "p95_ms": 642.5582220156988,
      "latency_unit": "document call",
      "latency_basis": "Primary runs; mini, Oct 3–4"
    },
    {
      "key": "clef_flash_batch_generic",
      "label": "Clef-flash · batch",
      "family": "clef",
      "quality": 0.2827373060086631,
      "per_corpus": {
        "scifact": 0.4433380438694286,
        "nfcorpus": 0.22762299003844674,
        "fiqa": 0.17725088411811402
      },
      "usd_per_1k": 0.5453999999999999,
      "p50_ms": 630.9843191411346,
      "p95_ms": 956.4967082502941,
      "latency_unit": "query call",
      "latency_basis": "Primary runs; mini, Oct 3–4"
    },
    {
      "key": "clef_flash_choice_generic",
      "label": "Clef-flash · choice",
      "family": "clef",
      "quality": 0.32886961563293543,
      "per_corpus": {
        "scifact": 0.5110381324488009,
        "nfcorpus": 0.25986664765822676,
        "fiqa": 0.2157040667917786
      },
      "usd_per_1k": 0.25227,
      "p50_ms": 410.1440904972454,
      "p95_ms": 621.3712911121547,
      "latency_unit": "query call",
      "latency_basis": "Primary runs; mini, Oct 3–4"
    },
    {
      "key": "clef_pair_generic",
      "label": "Clef · pair",
      "family": "clef",
      "quality": 0.49408689777838255,
      "per_corpus": {
        "scifact": 0.75884107233817,
        "nfcorpus": 0.34451894949751655,
        "fiqa": 0.37890067149946105
      },
      "usd_per_1k": 3.694442534571723,
      "p50_ms": 482.17724977682036,
      "p95_ms": 986.2054859598478,
      "latency_unit": "document call",
      "latency_basis": "Primary runs; mini, Oct 3–4"
    },
    {
      "key": "pplx_decider_batch_generic",
      "label": "Perplexity · batch",
      "family": "pplx",
      "quality": 0.4996382704536245,
      "per_corpus": {
        "scifact": 0.7697435540120146,
        "nfcorpus": 0.3523659908017212,
        "fiqa": 0.37680526654713786
      },
      "usd_per_1k": 11.588956194014447,
      "p50_ms": 1782.91133333308,
      "p95_ms": 2770.149055402726,
      "latency_unit": "query call",
      "latency_basis": "Primary runs; mini, Oct 3–4"
    },
    {
      "key": "pplx_decider_choice_generic",
      "label": "Perplexity · choice",
      "family": "pplx",
      "quality": 0.5014512822320403,
      "per_corpus": {
        "scifact": 0.7760365238734583,
        "nfcorpus": 0.34678496424875166,
        "fiqa": 0.38153235857391093
      },
      "usd_per_1k": 0.3969785398004816,
      "p50_ms": 521.4201177780827,
      "p95_ms": 737.407041558375,
      "latency_unit": "query call",
      "latency_basis": "Primary runs; mini, Oct 3–4"
    },
    {
      "key": "pplx_decider_pair_generic",
      "label": "Perplexity · pair",
      "family": "pplx",
      "quality": 0.4896610812564139,
      "per_corpus": {
        "scifact": 0.7517977777030154,
        "nfcorpus": 0.3434656895410013,
        "fiqa": 0.37371977652522503
      },
      "usd_per_1k": 0.5682172963192295,
      "p50_ms": 215.59583310348293,
      "p95_ms": 321.2323891930282,
      "latency_unit": "document call",
      "latency_basis": "Primary runs; mini, Oct 3–4"
    }
  ],
  "matched_latency": {
    "quality_basis": "Full 300-query SciFact quality",
    "latency_basis": "Same 50 SciFact queries; mini; Oct 4 04:51–04:58 UTC; concurrency 4; 0.4 request starts/sec; queue delay excluded",
    "rows": [
      {
        "key": "pplx_decider_batch_generic",
        "label": "Perplexity · batch",
        "family": "pplx",
        "quality": 0.7697435540120146,
        "p50_ms": 2008.119708392769,
        "p95_ms": 3181.7547087557614
      },
      {
        "key": "jev_batch_generic",
        "label": "Jev · batch",
        "family": "jev",
        "quality": 0.7683290081756676,
        "p50_ms": 192.45477113872766,
        "p95_ms": 233.0449167639017
      },
      {
        "key": "clef_flash_batch_generic",
        "label": "Clef-flash · batch",
        "family": "clef",
        "quality": 0.4433380438694286,
        "p50_ms": 530.1146672572941,
        "p95_ms": 884.5063331536949
      }
    ]
  },
  "bm25_per_corpus": {
    "scifact": 0.667,
    "nfcorpus": 0.31,
    "fiqa": 0.234
  }
}