{
 "eval_result_sha": "975139780665fb1e2c6f23fecd32f361845a75ff",
 "eval_source_sha": "6b2f65611d523e21b45991903dd7d48a1f772d27",
 "api_window_utc": [
  "2026-10-24T13:36:15Z",
  "2026-10-24T13:58:58Z"
 ],
 "prices_usd_per_m_input": {
  "pplx-decider-v1.2": 0.02,
  "pplx-decider-flash-v1.2": 0.01,
  "pplx-decider-v1-27b (Oct 3-4 runs)": 0.04
 },
 "quality_metric": "nDCG@10, linear gain, trec_eval tie-break; unweighted mean of SciFact(300)/NFCorpus(323)/FiQA(300 sample)",
 "independent_recompute": "nDCG@10 (linear gain, trec_eval tie-break), costs and paired bootstrap (seed 12345, 4,000 draws) recomputed from the raw run files; matches the eval to 4 decimal places",
 "new": {
  "pplx-decider-v1.2_choice": {
   "per_corpus": {
    "scifact": 0.7791315841203013,
    "nfcorpus": 0.3574565625731375,
    "fiqa": 0.39568181370489885
   },
   "coverage": {
    "scifact": 300,
    "nfcorpus": 323,
    "fiqa": 300
   },
   "quality": 0.5107566534661125,
   "usd_per_1k": 0.1984292699002408,
   "input_tokens_per_query": {
    "scifact": 11165.5,
    "nfcorpus": 11221.037151702787,
    "fiqa": 7377.8533333333335
   },
   "p50_ms": 479.25506266377243,
   "p95_ms": 591.2874670078357,
   "p50_ms_per_corpus": {
    "scifact": 510.3916575317271,
    "nfcorpus": 516.1143869627267,
    "fiqa": 411.25914349686354
   },
   "p95_ms_per_corpus": {
    "scifact": 647.2015840699896,
    "nfcorpus": 590.5883399536833,
    "fiqa": 536.0724769998342
   },
   "returned_models": [
    "pplx-decider-v1.2"
   ],
   "label": "Perplexity v1.2 27B · choice",
   "model": "pplx-decider-v1.2",
   "shape": "choice",
   "full": true,
   "latency_basis": "Linux server, Oct 24, concurrency 4, ≤5 request starts/s; successful requests only; per query call"
  },
  "pplx-decider-v1.2_batch": {
   "per_corpus": {
    "scifact": 0.7665569870587879,
    "nfcorpus": 0.3438686030320072,
    "fiqa": 0.38369687156770177
   },
   "coverage": {
    "scifact": 300,
    "nfcorpus": 323,
    "fiqa": 300
   },
   "quality": 0.49804082055283233,
   "usd_per_1k": 0.2462692699002408,
   "input_tokens_per_query": {
    "scifact": 13557.5,
    "nfcorpus": 13613.037151702787,
    "fiqa": 9769.853333333333
   },
   "p50_ms": 539.6659496667174,
   "p95_ms": 662.2537353153651,
   "p50_ms_per_corpus": {
    "scifact": 564.3573984852992,
    "nfcorpus": 577.6395120192319,
    "fiqa": 477.0009384956211
   },
   "p95_ms_per_corpus": {
    "scifact": 734.7129399422556,
    "nfcorpus": 693.3154229773208,
    "fiqa": 558.7328430265188
   },
   "returned_models": [
    "pplx-decider-v1.2"
   ],
   "label": "Perplexity v1.2 27B · batch",
   "model": "pplx-decider-v1.2",
   "shape": "batch",
   "full": true,
   "latency_basis": "Linux server, Oct 24, concurrency 4, ≤5 request starts/s; successful requests only; per query call"
  },
  "pplx-decider-v1.2_pair": {
   "per_corpus": {
    "scifact": 0.8508145068909506,
    "nfcorpus": 0.31170597867262345,
    "fiqa": 0.17410277444742733
   },
   "coverage": {
    "scifact": 10,
    "nfcorpus": 10,
    "fiqa": 10
   },
   "quality": 0.4455410866703338,
   "usd_per_1k": 0.277306,
   "input_tokens_per_query": {
    "scifact": 14866.6,
    "nfcorpus": 15113.5,
    "fiqa": 11615.8
   },
   "p50_ms": 241.67998464933285,
   "p95_ms": 299.6178299654275,
   "p50_ms_per_corpus": {
    "scifact": 244.23064041184261,
    "nfcorpus": 241.5712540387176,
    "fiqa": 239.23805949743837
   },
   "p95_ms_per_corpus": {
    "scifact": 306.4179759239778,
    "nfcorpus": 296.248028986156,
    "fiqa": 296.1874849861488
   },
   "returned_models": [
    "pplx-decider-v1.2"
   ],
   "label": "Perplexity v1.2 27B · pair",
   "model": "pplx-decider-v1.2",
   "shape": "pair",
   "full": false,
   "latency_basis": "Linux server, Oct 24, concurrency 4, ≤5 request starts/s; successful requests only; per document call"
  },
  "pplx-decider-flash-v1.2_choice": {
   "per_corpus": {
    "scifact": 0.7713598759339452,
    "nfcorpus": 0.3540923295129741,
    "fiqa": 0.37523792404129463
   },
   "coverage": {
    "scifact": 300,
    "nfcorpus": 323,
    "fiqa": 300
   },
   "quality": 0.500230043162738,
   "usd_per_1k": 0.0992146349501204,
   "input_tokens_per_query": {
    "scifact": 11165.5,
    "nfcorpus": 11221.037151702787,
    "fiqa": 7377.8533333333335
   },
   "p50_ms": 311.4774420197743,
   "p95_ms": 472.4936456186697,
   "p50_ms_per_corpus": {
    "scifact": 328.2432515406981,
    "nfcorpus": 338.7947790324688,
    "fiqa": 267.3942954861559
   },
   "p95_ms_per_corpus": {
    "scifact": 417.362225940451,
    "nfcorpus": 420.78198795206845,
    "fiqa": 579.3367229634896
   },
   "returned_models": [
    "pplx-decider-flash-v1.2"
   ],
   "label": "Perplexity v1.2 flash · choice",
   "model": "pplx-decider-flash-v1.2",
   "shape": "choice",
   "full": true,
   "latency_basis": "Linux server, Oct 24, concurrency 4, ≤5 request starts/s; successful requests only; per query call"
  },
  "pplx-decider-flash-v1.2_batch": {
   "per_corpus": {
    "scifact": 0.7433225588267796,
    "nfcorpus": 0.33020462462990696,
    "fiqa": 0.2773681357161287
   },
   "coverage": {
    "scifact": 16,
    "nfcorpus": 16,
    "fiqa": 20
   },
   "quality": 0.45029843972427175,
   "usd_per_1k": 2.8347075000000004,
   "input_tokens_per_query": {
    "scifact": 314002.5,
    "nfcorpus": 320756.25,
    "fiqa": 215653.5
   },
   "p50_ms": 3528.8945219751135,
   "p95_ms": 4613.612223688203,
   "p50_ms_per_corpus": {
    "scifact": 3825.1765784807503,
    "nfcorpus": 4073.819506971631,
    "fiqa": 2687.6874804729596
   },
   "p95_ms_per_corpus": {
    "scifact": 5339.354344061576,
    "nfcorpus": 5184.601958026178,
    "fiqa": 3316.880368976854
   },
   "returned_models": [
    "pplx-decider-flash-v1.2"
   ],
   "label": "Perplexity v1.2 flash · batch",
   "model": "pplx-decider-flash-v1.2",
   "shape": "batch",
   "full": false,
   "latency_basis": "Linux server, Oct 24, concurrency 4, ≤5 request starts/s; successful requests only; per query call"
  },
  "pplx-decider-flash-v1.2_pair": {
   "per_corpus": {
    "scifact": 0.8428652643116484,
    "nfcorpus": 0.316658289793172,
    "fiqa": 0.15637884576902256
   },
   "coverage": {
    "scifact": 10,
    "nfcorpus": 10,
    "fiqa": 10
   },
   "quality": 0.43863413329128104,
   "usd_per_1k": 0.138653,
   "input_tokens_per_query": {
    "scifact": 14866.6,
    "nfcorpus": 15113.5,
    "fiqa": 11615.8
   },
   "p50_ms": 160.88428033981472,
   "p95_ms": 187.25281798591217,
   "p50_ms_per_corpus": {
    "scifact": 160.85518553154543,
    "nfcorpus": 159.54293549293652,
    "fiqa": 162.25471999496222
   },
   "p95_ms_per_corpus": {
    "scifact": 187.36858794,
    "nfcorpus": 185.99716702010483,
    "fiqa": 188.39269899763167
   },
   "returned_models": [
    "pplx-decider-flash-v1.2"
   ],
   "label": "Perplexity v1.2 flash · pair",
   "model": "pplx-decider-flash-v1.2",
   "shape": "pair",
   "full": false,
   "latency_basis": "Linux server, Oct 24, concurrency 4, ≤5 request starts/s; successful requests only; per document call"
  }
 },
 "historical_rows": [
  {
   "key": "jev_batch_generic",
   "label": "Jev · batch",
   "family": "jev",
   "quality": 0.5008547536859029,
   "per_corpus": {
    "scifact": 0.7683290081756676,
    "nfcorpus": 0.35820059943282617,
    "fiqa": 0.37603465344921505
   },
   "usd_per_1k": 0.5372456140350877,
   "p50_ms": 223.14528624216715,
   "p95_ms": 1419.498046239217,
   "latency_unit": "query call",
   "latency_basis": "Historical runs; laptop, September"
  },
  {
   "key": "jev_pair_generic",
   "label": "Jev · pair",
   "family": "jev",
   "quality": 0.5021354827688203,
   "per_corpus": {
    "scifact": 0.7721709227158036,
    "nfcorpus": 0.35799232186049085,
    "fiqa": 0.3762432037301664
   },
   "usd_per_1k": 0.8670567595459235,
   "p50_ms": 131.45633538564047,
   "p95_ms": 255.9018929799398,
   "latency_unit": "document call",
   "latency_basis": "Historical runs; laptop, September"
  },
  {
   "key": "mixedbread_v31",
   "label": "Mixedbread v3.1 listwise",
   "family": "historical",
   "quality": 0.510609841223261,
   "per_corpus": {
    "scifact": 0.7674914198991412,
    "nfcorpus": 0.3666336737446514,
    "fiqa": 0.39770443002599043
   },
   "usd_per_1k": 0.272427,
   "p50_ms": 216.68759981791177,
   "p95_ms": 262.3496850331624,
   "latency_unit": "query call",
   "latency_basis": "Historical runs; laptop, September",
   "usd_per_1k_original_snapshot": 1.41925,
   "price_basis": {
    "method": "estimate",
    "billed_usage": null,
    "estimated_on": "2026-10-24",
    "rate_usd_per_m_input": 0.03,
    "rate_source": "Approximate per-token rate given by Mixedbread on 2026-10-24 (about $0.03 per million input tokens); not a published price page.",
    "tokenizer": "Qwen/Qwen3.5-27B at revision fc05daec18b0a78c049392ed2e771dde82bdf654, tokenizer.json sha256 5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42 (the same file ships with Qwen3.5-0.8B, 4B, 9B, 35B-A3B and 397B-A17B)",
    "inputs": "The frozen BM25 top-30 shortlists the v3.1 run reranked, exactly as sent: the query plus 30 documents, each document as title + '. ' + text, no truncation.",
    "counting": "Listwise lexical count: query tokenized once plus each document tokenized separately, no special tokens. Server-side wrapper, template or special tokens are unknown and not counted; each extra token per document would add about $0.0009 per 1,000 queries.",
    "tokens_per_query": {
     "scifact": 10323.76,
     "nfcorpus": 10376.24,
     "fiqa": 6542.66
    },
    "usd_per_1k_per_corpus": {
     "scifact": 0.309713,
     "nfcorpus": 0.311287,
     "fiqa": 0.19628
    },
    "aggregate": "Unweighted mean of the three corpora, as for quality.",
    "sensitivity_query_counted_per_document_usd_per_1k": 0.283528,
    "superseded": "The original snapshot value (usd_per_1k_original_snapshot) counted large-v2 tokenizer tokens with the query once per document at an earlier per-token rate; it is kept for provenance and no longer used."
   }
  },
  {
   "key": "voyage3",
   "label": "Voyage rerank-3",
   "family": "historical",
   "quality": 0.504438416759762,
   "per_corpus": {
    "scifact": 0.7546579707306361,
    "nfcorpus": 0.35669798964188537,
    "fiqa": 0.40195928990676455
   },
   "usd_per_1k": 0.5,
   "p50_ms": 185.2254867553711,
   "p95_ms": 286.7468198140462,
   "latency_unit": "query call",
   "latency_basis": "Historical runs; laptop, September"
  },
  {
   "key": "cohere",
   "label": "Cohere v3.5",
   "family": "historical",
   "quality": 0.48614827536480154,
   "per_corpus": {
    "scifact": 0.7454041499445426,
    "nfcorpus": 0.3394673420129891,
    "fiqa": 0.37357333413687294
   },
   "usd_per_1k": 2,
   "p50_ms": 191.98481241861978,
   "p95_ms": 456.168254216512,
   "latency_unit": "query call",
   "latency_basis": "Historical runs; laptop, September"
  },
  {
   "key": "mixedbread",
   "label": "Mixedbread large-v2",
   "family": "historical",
   "quality": 0.4755640209697933,
   "per_corpus": {
    "scifact": 0.7491112976585692,
    "nfcorpus": 0.32382252788004123,
    "fiqa": 0.3537582373707694
   },
   "usd_per_1k": 1.41925,
   "p50_ms": 360.6390953063965,
   "p95_ms": 428.5889466603597,
   "latency_unit": "query call",
   "latency_basis": "Historical runs; laptop, September",
   "price_basis": {
    "method": "estimate",
    "billed_usage": null,
    "note": "Historical value from the original snapshot (large-v2 tokenizer counts at the rate quoted when it ran), not re-estimated; large-v2 is not shown in this post."
   }
  },
  {
   "key": "jina35",
   "label": "Jina v3.5*",
   "family": "historical",
   "quality": 0.4917333333333333,
   "per_corpus": {
    "scifact": 0.7505,
    "nfcorpus": 0.3449,
    "fiqa": 0.3798
   },
   "usd_per_1k": 0.47308333333333336,
   "p50_ms": 302.74200439453125,
   "p95_ms": 606.8589687347412,
   "latency_unit": "query call",
   "latency_basis": "Historical API SciFact only; quality from open weights"
  },
  {
   "key": "clef_flash_pair_generic",
   "label": "Clef-flash · pair",
   "family": "clef",
   "quality": 0.49832834925328084,
   "per_corpus": {
    "scifact": 0.7686173593041452,
    "nfcorpus": 0.35483538031282263,
    "fiqa": 0.3715323081428747
   },
   "usd_per_1k": 1.3854159504643964,
   "p50_ms": 262.52713869325817,
   "p95_ms": 642.5582220156988,
   "latency_unit": "document call",
   "latency_basis": "Primary runs; Mac mini, Oct 3–4"
  },
  {
   "key": "clef_flash_batch_generic",
   "label": "Clef-flash · batch",
   "family": "clef",
   "quality": 0.2827373060086631,
   "per_corpus": {
    "scifact": 0.4433380438694286,
    "nfcorpus": 0.22762299003844674,
    "fiqa": 0.17725088411811402
   },
   "usd_per_1k": 0.5453999999999999,
   "p50_ms": 630.9843191411346,
   "p95_ms": 956.4967082502941,
   "latency_unit": "query call",
   "latency_basis": "Primary runs; Mac mini, Oct 3–4"
  },
  {
   "key": "clef_flash_choice_generic",
   "label": "Clef-flash · choice",
   "family": "clef",
   "quality": 0.32886961563293543,
   "per_corpus": {
    "scifact": 0.5110381324488009,
    "nfcorpus": 0.25986664765822676,
    "fiqa": 0.2157040667917786
   },
   "usd_per_1k": 0.25227,
   "p50_ms": 410.1440904972454,
   "p95_ms": 621.3712911121547,
   "latency_unit": "query call",
   "latency_basis": "Primary runs; Mac mini, Oct 3–4"
  },
  {
   "key": "clef_pair_generic",
   "label": "Clef · pair",
   "family": "clef",
   "quality": 0.49408689777838255,
   "per_corpus": {
    "scifact": 0.75884107233817,
    "nfcorpus": 0.34451894949751655,
    "fiqa": 0.37890067149946105
   },
   "usd_per_1k": 3.694442534571723,
   "p50_ms": 482.17724977682036,
   "p95_ms": 986.2054859598478,
   "latency_unit": "document call",
   "latency_basis": "Primary runs; Mac mini, Oct 3–4"
  },
  {
   "key": "pplx_decider_batch_generic",
   "label": "Perplexity · batch",
   "family": "pplx",
   "quality": 0.4996382704536245,
   "per_corpus": {
    "scifact": 0.7697435540120146,
    "nfcorpus": 0.3523659908017212,
    "fiqa": 0.37680526654713786
   },
   "usd_per_1k": 11.588956194014447,
   "p50_ms": 1782.91133333308,
   "p95_ms": 2770.149055402726,
   "latency_unit": "query call",
   "latency_basis": "Primary runs; Mac mini, Oct 3–4"
  },
  {
   "key": "pplx_decider_choice_generic",
   "label": "Perplexity · choice",
   "family": "pplx",
   "quality": 0.5014512822320403,
   "per_corpus": {
    "scifact": 0.7760365238734583,
    "nfcorpus": 0.34678496424875166,
    "fiqa": 0.38153235857391093
   },
   "usd_per_1k": 0.3969785398004816,
   "p50_ms": 521.4201177780827,
   "p95_ms": 737.407041558375,
   "latency_unit": "query call",
   "latency_basis": "Primary runs; Mac mini, Oct 3–4"
  },
  {
   "key": "pplx_decider_pair_generic",
   "label": "Perplexity · pair",
   "family": "pplx",
   "quality": 0.4896610812564139,
   "per_corpus": {
    "scifact": 0.7517977777030154,
    "nfcorpus": 0.3434656895410013,
    "fiqa": 0.37371977652522503
   },
   "usd_per_1k": 0.5682172963192295,
   "p50_ms": 215.59583310348293,
   "p95_ms": 321.2323891930282,
   "latency_unit": "document call",
   "latency_basis": "Primary runs; Mac mini, Oct 3–4"
  },
  {
   "key": "openai_decisions_luna_choice_generic",
   "label": "OpenAI · choice",
   "family": "openai",
   "quality": 0.4720718932198098,
   "per_corpus": {
    "scifact": 0.7506458856327863,
    "nfcorpus": 0.3108589607943464,
    "fiqa": 0.35471083323229663
   },
   "usd_per_1k": 0.9189763168214654,
   "usd_per_1k_per_corpus": {
    "scifact": 1.0382726666666666,
    "nfcorpus": 1.0408919504643963,
    "fiqa": 0.6777643333333333
   },
   "p50_ms": 255.49921509809792,
   "p95_ms": 497.89079107965034,
   "latency_unit": "query call",
   "latency_basis": "Primary runs; Mac mini, Oct 7; concurrency 6; mean of three corpora"
  },
  {
   "key": "openai_decisions_luna_batch_generic",
   "label": "OpenAI · batch",
   "family": "openai",
   "quality": 0.45013299681773233,
   "per_corpus": {
    "scifact": 0.6806812243596224,
    "nfcorpus": 0.3234378761646755,
    "fiqa": 0.346279889928899
   },
   "usd_per_1k": 1.5533763168214658,
   "usd_per_1k_per_corpus": {
    "scifact": 1.672672666666667,
    "nfcorpus": 1.6752919504643966,
    "fiqa": 1.3121643333333333
   },
   "p50_ms": 345.51152780962485,
   "p95_ms": 854.7872918037077,
   "latency_unit": "query call",
   "latency_basis": "Primary runs; Mac mini, Oct 7; concurrency 6; mean of three corpora"
  },
  {
   "key": "openai_decisions_luna_pair_generic",
   "label": "OpenAI · pair",
   "family": "openai",
   "quality": 0.4409926621433682,
   "per_corpus": {
    "scifact": 0.6704095734827278,
    "nfcorpus": 0.30628232606164246,
    "fiqa": 0.3462860868857342
   },
   "usd_per_1k": 1.5803283071895426,
   "usd_per_1k_per_corpus": {
    "scifact": 1.7182706666666665,
    "nfcorpus": 1.6819705882352942,
    "fiqa": 1.340743666666667
   },
   "p50_ms": 205.29084016258517,
   "p95_ms": 451.08506937200826,
   "latency_unit": "30 calls",
   "latency_basis": "Primary runs; Mac mini, Oct 7; per pair call (not per query)"
  }
 ],
 "historical_source_commit": "6b2f65611d523e21b45991903dd7d48a1f772d27",
 "bm25": 0.4039205123793589,
 "bm25_per_corpus": {
  "scifact": 0.6669140404875069,
  "nfcorpus": 0.3103988583346401,
  "fiqa": 0.23444863831592963
 },
 "paired": {
  "v12_choice|rerank_mixedbread_v31": {
   "scifact": {
    "n": 300,
    "delta": 0.011640164221160694,
    "ci": [
     0.0021146761430205436,
     0.02185115693448621
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.009177111171513855,
    "ci": [
     -0.016086711091003875,
     -0.001942925904314785
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.002022616321091301,
    "ci": [
     -0.01069563451620855,
     0.006565545449452915
    ]
   },
   "mean3": {
    "delta": 0.00014681224285184577,
    "ci": [
     -0.004744762974070792,
     0.005220217392133029
    ]
   }
  },
  "v12_choice|rerank_voyage3": {
   "scifact": {
    "n": 300,
    "delta": 0.02447361338966565,
    "ci": [
     0.01135056225180499,
     0.03813890279054318
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.0007585729312521827,
    "ci": [
     -0.007340111927066237,
     0.009291238302387338
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.006277476201865381,
    "ci": [
     -0.016010392444172482,
     0.0032249751387676538
    ]
   },
   "mean3": {
    "delta": 0.006318236706350816,
    "ci": [
     0.00035451515903307834,
     0.012605741362852986
    ]
   }
  },
  "v12_choice|rerank_cohere": {
   "scifact": {
    "n": 300,
    "delta": 0.0337274341757592,
    "ci": [
     0.016873715505825754,
     0.050866839196569306
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.017989220560148414,
    "ci": [
     0.00906459965799583,
     0.02705823194838349
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.022108479568026227,
    "ci": [
     0.0084956565078663,
     0.03609378863869505
    ]
   },
   "mean3": {
    "delta": 0.02460837810131128,
    "ci": [
     0.016721876543840594,
     0.032313249573846424
    ]
   }
  },
  "v12_choice|jev_batch_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.010802575944634208,
    "ci": [
     -0.0026522708491448755,
     0.024646885931821173
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.0007440368596886098,
    "ci": [
     -0.00971074533007056,
     0.008566486018870712
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.01964716025568408,
    "ci": [
     0.006819386547771909,
     0.032805137247320044
    ]
   },
   "mean3": {
    "delta": 0.009901899780209892,
    "ci": [
     0.0030388489554896386,
     0.017028082827138944
    ]
   }
  },
  "v12_choice|jev_pair_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.006960661404498195,
    "ci": [
     -0.007400543216151556,
     0.022244960194057074
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.0005357592873533019,
    "ci": [
     -0.009080382218833725,
     0.008650293542174432
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.019438609974732745,
    "ci": [
     0.004388964157745406,
     0.0343057091096369
    ]
   },
   "mean3": {
    "delta": 0.008621170697292546,
    "ci": [
     0.001250760521875122,
     0.01627059421792619
    ]
   }
  },
  "v12_choice|pplx_decider_choice_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.003095060246842909,
    "ci": [
     -0.007708090759047363,
     0.014008366658225534
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.010671598324385656,
    "ci": [
     0.00433782235677297,
     0.016812397948039806
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.014149455130987774,
    "ci": [
     0.006345557732837546,
     0.02273655804040667
    ]
   },
   "mean3": {
    "delta": 0.009305371234072114,
    "ci": [
     0.004305934565241975,
     0.01442521899011879
    ]
   }
  },
  "v12_choice|pplx_decider_batch_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.009388030108287176,
    "ci": [
     -0.004013761560699324,
     0.022935237411926223
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.005090571771416308,
    "ci": [
     -0.0032324591562074563,
     0.014264253625576038
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.018876547157761003,
    "ci": [
     0.00944778874606044,
     0.028888884052579774
    ]
   },
   "mean3": {
    "delta": 0.011118383012488164,
    "ci": [
     0.004925373855859133,
     0.01757020208683186
    ]
   }
  },
  "v12_choice|clef_flash_pair_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.010514224816156614,
    "ci": [
     -0.003014237957450416,
     0.02456349790809758
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.002621182260314915,
    "ci": [
     -0.005719460803154802,
     0.011461972461445348
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.024149505562024484,
    "ci": [
     0.012438171150933078,
     0.03637253774959781
    ]
   },
   "mean3": {
    "delta": 0.012428304212832003,
    "ci": [
     0.005700863795612601,
     0.019260874905196245
    ]
   }
  },
  "v12_choice|openai_decisions_luna_choice_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.02848569848751554,
    "ci": [
     0.013974284629195444,
     0.04386952161634732
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.04650838642051096,
    "ci": [
     0.03534816234576798,
     0.05806291376539826
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.040970980472602524,
    "ci": [
     0.02819813633300476,
     0.05477376419520992
    ]
   },
   "mean3": {
    "delta": 0.038655021793543014,
    "ci": [
     0.030991061104542406,
     0.04628414783880613
    ]
   }
  },
  "v12_choice|bm25": {
   "scifact": {
    "n": 300,
    "delta": 0.11221754363279494,
    "ci": [
     0.0833411129618769,
     0.1416587253960913
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.04705770423849741,
    "ci": [
     0.03399335751349051,
     0.06086587267581576
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.16123317538896953,
    "ci": [
     0.13406023891322805,
     0.190266451369426
    ]
   },
   "mean3": {
    "delta": 0.10683614108675395,
    "ci": [
     0.09280873282106579,
     0.12105488664591309
    ]
   }
  },
  "v12_choice|v12_batch": {
   "scifact": {
    "n": 300,
    "delta": 0.012574597061513561,
    "ci": [
     -0.00044825719758410633,
     0.026855730112365316
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.013587959541130214,
    "ci": [
     0.003113331894083855,
     0.024425713662428235
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.011984942137196978,
    "ci": [
     0.002783503302702959,
     0.021272735926223172
    ]
   },
   "mean3": {
    "delta": 0.012715832913280252,
    "ci": [
     0.006355443442333536,
     0.019355340645509356
    ]
   }
  },
  "v12_choice|flash_choice": {
   "scifact": {
    "n": 300,
    "delta": 0.007771708186356274,
    "ci": [
     -0.00570980543107685,
     0.021639381516679324
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.0033642330601632263,
    "ci": [
     -0.003690062856766621,
     0.010787068043694227
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.02044388966360404,
    "ci": [
     0.008464148124730328,
     0.032460303460416735
    ]
   },
   "mean3": {
    "delta": 0.010526610303374512,
    "ci": [
     0.004137853311545391,
     0.01709195704710131
    ]
   }
  },
  "v12_batch|rerank_mixedbread_v31": {
   "scifact": {
    "n": 300,
    "delta": -0.0009344328403528672,
    "ci": [
     -0.014333921285056712,
     0.011803496091518581
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.022765070712644053,
    "ci": [
     -0.033287544743190234,
     -0.012778688207652166
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.014007558458288277,
    "ci": [
     -0.023370529966546216,
     -0.004717147781623496
    ]
   },
   "mean3": {
    "delta": -0.012569020670428399,
    "ci": [
     -0.01908671188098078,
     -0.006422998989929113
    ]
   }
  },
  "v12_batch|rerank_voyage3": {
   "scifact": {
    "n": 300,
    "delta": 0.011899016328152092,
    "ci": [
     -0.00343372028822332,
     0.02694545876714405
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.012829386609878035,
    "ci": [
     -0.023753517396926844,
     -0.0025062540113307426
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.01826241833906235,
    "ci": [
     -0.031082291334133812,
     -0.005792679037484006
    ]
   },
   "mean3": {
    "delta": -0.00639759620692943,
    "ci": [
     -0.014145937523396724,
     0.0007691408909149393
    ]
   }
  },
  "v12_batch|rerank_cohere": {
   "scifact": {
    "n": 300,
    "delta": 0.02115283711424564,
    "ci": [
     0.0009967875635003168,
     0.0418208041175033
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.0044012610190182055,
    "ci": [
     -0.00657884505935324,
     0.0152461105872238
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.010123537430829247,
    "ci": [
     -0.004035393138974275,
     0.024631824568873133
    ]
   },
   "mean3": {
    "delta": 0.011892545188031033,
    "ci": [
     0.0027034790153535296,
     0.02070819535053105
    ]
   }
  },
  "v12_batch|jev_batch_generic": {
   "scifact": {
    "n": 300,
    "delta": -0.0017720211168793525,
    "ci": [
     -0.015086521480681521,
     0.01117861569150129
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.014331996400818822,
    "ci": [
     -0.02489334762356949,
     -0.004247643455052727
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.007662218118487099,
    "ci": [
     -0.004060780073846228,
     0.020096117849549233
    ]
   },
   "mean3": {
    "delta": -0.0028139331330703584,
    "ci": [
     -0.009789998175973395,
     0.004229285147722783
    ]
   }
  },
  "v12_batch|jev_pair_generic": {
   "scifact": {
    "n": 300,
    "delta": -0.005613935657015367,
    "ci": [
     -0.021864030588750177,
     0.010663317505435614
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.014123718828483522,
    "ci": [
     -0.024886583074100008,
     -0.0038092661832476496
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.007453667837535762,
    "ci": [
     -0.005337206877954778,
     0.02030359801912681
    ]
   },
   "mean3": {
    "delta": -0.004094662215987708,
    "ci": [
     -0.011688706216033247,
     0.0036766477389767217
    ]
   }
  },
  "v12_batch|pplx_decider_choice_generic": {
   "scifact": {
    "n": 300,
    "delta": -0.009479536814670647,
    "ci": [
     -0.02338380111195938,
     0.003271503110989503
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.0029163612167445625,
    "ci": [
     -0.01544222393808724,
     0.009020826192813873
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.0021645129937907935,
    "ci": [
     -0.008971845831349011,
     0.013290769721272253
    ]
   },
   "mean3": {
    "delta": -0.003410461679208139,
    "ci": [
     -0.010627228735994558,
     0.003291299201835733
    ]
   }
  },
  "v12_batch|pplx_decider_batch_generic": {
   "scifact": {
    "n": 300,
    "delta": -0.00318656695322638,
    "ci": [
     -0.015539989118936448,
     0.008303787436173837
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.0084973877697139,
    "ci": [
     -0.018586160371025617,
     0.0014097709938950104
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.006891605020564033,
    "ci": [
     -0.0031951595217245008,
     0.017370139687298655
    ]
   },
   "mean3": {
    "delta": -0.0015974499007920823,
    "ci": [
     -0.007892774317990766,
     0.00453819300314023
    ]
   }
  },
  "v12_batch|clef_flash_pair_generic": {
   "scifact": {
    "n": 300,
    "delta": -0.0020603722453569452,
    "ci": [
     -0.015179987167895858,
     0.010781228340289621
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.0109667772808153,
    "ci": [
     -0.022043196657993203,
     -0.0010181544057768568
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.012164563424827503,
    "ci": [
     -0.0018177021883993805,
     0.026559526102733023
    ]
   },
   "mean3": {
    "delta": -0.00028752870044824785,
    "ci": [
     -0.007738653153057957,
     0.006943662692714156
    ]
   }
  },
  "v12_batch|openai_decisions_luna_choice_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.015911101426001974,
    "ci": [
     -0.0004943457876461784,
     0.03230978027503214
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.032920426879380765,
    "ci": [
     0.01804008080799782,
     0.04732404214531342
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.028986038335405542,
    "ci": [
     0.014227448368805138,
     0.04408165563177169
    ]
   },
   "mean3": {
    "delta": 0.025939188880262764,
    "ci": [
     0.01683010711223071,
     0.03451830990335269
    ]
   }
  },
  "v12_batch|bm25": {
   "scifact": {
    "n": 300,
    "delta": 0.09964294657128138,
    "ci": [
     0.0701516999145949,
     0.12952540783250382
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.0334697446973672,
    "ci": [
     0.016855970240437362,
     0.04960662510581304
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.1492482332517725,
    "ci": [
     0.11998997860508823,
     0.18020807141760106
    ]
   },
   "mean3": {
    "delta": 0.09412030817347368,
    "ci": [
     0.07879534563039813,
     0.10873956711597209
    ]
   }
  },
  "v12_batch|v12_choice": {
   "scifact": {
    "n": 300,
    "delta": -0.012574597061513561,
    "ci": [
     -0.026855730112365316,
     0.00044825719758410633
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.013587959541130214,
    "ci": [
     -0.024425713662428235,
     -0.003113331894083855
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.011984942137196978,
    "ci": [
     -0.021272735926223172,
     -0.002783503302702959
    ]
   },
   "mean3": {
    "delta": -0.012715832913280252,
    "ci": [
     -0.019355340645509356,
     -0.006355443442333536
    ]
   }
  },
  "v12_batch|flash_choice": {
   "scifact": {
    "n": 300,
    "delta": -0.004802888875157289,
    "ci": [
     -0.01907232288298107,
     0.008796397198375988
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.010223726480966991,
    "ci": [
     -0.019699609800694916,
     -0.0007607675600003981
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.00845894752640707,
    "ci": [
     -0.004047176725747511,
     0.021015969319745528
    ]
   },
   "mean3": {
    "delta": -0.0021892226099057364,
    "ci": [
     -0.00919203144124684,
     0.004674867516169616
    ]
   }
  },
  "flash_choice|rerank_mixedbread_v31": {
   "scifact": {
    "n": 300,
    "delta": 0.0038684560348044173,
    "ci": [
     -0.010207211889657423,
     0.0177421924071687
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.012541344231677088,
    "ci": [
     -0.020614619922810735,
     -0.005010037795310768
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.02246650598469534,
    "ci": [
     -0.03452516115274064,
     -0.010453136122663442
    ]
   },
   "mean3": {
    "delta": -0.010379798060522671,
    "ci": [
     -0.017035496807283574,
     -0.0037729928873435257
    ]
   }
  },
  "flash_choice|rerank_voyage3": {
   "scifact": {
    "n": 300,
    "delta": 0.016701905203309376,
    "ci": [
     0.0020169736458094454,
     0.03210147844666174
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.0026056601289110456,
    "ci": [
     -0.010739472477564373,
     0.005197467939999111
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.02672136586546942,
    "ci": [
     -0.04033455728126287,
     -0.014155198610310279
    ]
   },
   "mean3": {
    "delta": -0.004208373597023697,
    "ci": [
     -0.01124411727751931,
     0.0027118332205634276
    ]
   }
  },
  "flash_choice|rerank_cohere": {
   "scifact": {
    "n": 300,
    "delta": 0.025955725989402916,
    "ci": [
     0.0036391185284119034,
     0.047415901073526584
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.014624987499985188,
    "ci": [
     0.005464939344802255,
     0.02389906799553846
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.0016645899044221827,
    "ci": [
     -0.015134235103701004,
     0.018645984939264934
    ]
   },
   "mean3": {
    "delta": 0.014081767797936762,
    "ci": [
     0.004385590729014693,
     0.023511350810222976
    ]
   }
  },
  "flash_choice|jev_batch_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.003030867758277932,
    "ci": [
     -0.008954273826097526,
     0.015510696323477775
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.004108269919851837,
    "ci": [
     -0.01166094346303634,
     0.0036005335767395072
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.0007967294079199661,
    "ci": [
     -0.014053284067584357,
     0.013182853425105974
    ]
   },
   "mean3": {
    "delta": -0.0006247105231646238,
    "ci": [
     -0.007288157277329448,
     0.006170131093691966
    ]
   }
  },
  "flash_choice|jev_pair_generic": {
   "scifact": {
    "n": 300,
    "delta": -0.0008110467818580809,
    "ci": [
     -0.013426262942171061,
     0.013025096751747673
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.003899992347516526,
    "ci": [
     -0.011389723624241298,
     0.004229334074185712
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.0010052796888713042,
    "ci": [
     -0.015318530468383074,
     0.013737150447420993
    ]
   },
   "mean3": {
    "delta": -0.0019054396060819703,
    "ci": [
     -0.008646035742373248,
     0.005122122915094659
    ]
   }
  },
  "flash_choice|pplx_decider_choice_generic": {
   "scifact": {
    "n": 300,
    "delta": -0.004676647939513365,
    "ci": [
     -0.01709141335977216,
     0.007181616154934103
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.0073073652642224275,
    "ci": [
     -0.0016037661951211338,
     0.016066529327137657
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.006294434532616272,
    "ci": [
     -0.017814175269656018,
     0.0042509253011667495
    ]
   },
   "mean3": {
    "delta": -0.001221239069302403,
    "ci": [
     -0.007494130012393799,
     0.004968110375217136
    ]
   }
  },
  "flash_choice|pplx_decider_batch_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.001616321921930903,
    "ci": [
     -0.011517157082851478,
     0.01493065925773833
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.001726338711253085,
    "ci": [
     -0.006272206765768867,
     0.010077265065067564
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.0015673425058430341,
    "ci": [
     -0.014548922673474962,
     0.011280311608324508
    ]
   },
   "mean3": {
    "delta": 0.0005917727091136512,
    "ci": [
     -0.006097746807011074,
     0.007547113246104402
    ]
   }
  },
  "flash_choice|clef_flash_pair_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.0027425166298003373,
    "ci": [
     -0.013121247729647019,
     0.019049170877282088
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.0007430507998483119,
    "ci": [
     -0.009123392333150439,
     0.007585424994983884
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.003705615898420435,
    "ci": [
     -0.010850160204039547,
     0.018567396761759117
    ]
   },
   "mean3": {
    "delta": 0.0019016939094574868,
    "ci": [
     -0.005924711705147287,
     0.009709141069011112
    ]
   }
  },
  "flash_choice|openai_decisions_luna_choice_generic": {
   "scifact": {
    "n": 300,
    "delta": 0.02071399030115926,
    "ci": [
     0.004420633017311886,
     0.0364177528113724
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.04314415336034774,
    "ci": [
     0.03066901824964208,
     0.0555627557677589
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.02052709080899846,
    "ci": [
     0.007736562373736513,
     0.03403145119346512
    ]
   },
   "mean3": {
    "delta": 0.028128411490168485,
    "ci": [
     0.019765576612481014,
     0.036337586082936656
    ]
   }
  },
  "flash_choice|bm25": {
   "scifact": {
    "n": 300,
    "delta": 0.10444583544643868,
    "ci": [
     0.07386726060202732,
     0.1350663134677955
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.04369347117833423,
    "ci": [
     0.028690794187072606,
     0.05864714995616738
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": 0.14078928572536553,
    "ci": [
     0.11309206447844074,
     0.1705450201781561
    ]
   },
   "mean3": {
    "delta": 0.09630953078337949,
    "ci": [
     0.08124203968418812,
     0.11127539410136979
    ]
   }
  },
  "flash_choice|v12_choice": {
   "scifact": {
    "n": 300,
    "delta": -0.007771708186356274,
    "ci": [
     -0.021639381516679324,
     0.00570980543107685
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": -0.0033642330601632263,
    "ci": [
     -0.010787068043694227,
     0.003690062856766621
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.02044388966360404,
    "ci": [
     -0.032460303460416735,
     -0.008464148124730328
    ]
   },
   "mean3": {
    "delta": -0.010526610303374512,
    "ci": [
     -0.01709195704710131,
     -0.004137853311545391
    ]
   }
  },
  "flash_choice|v12_batch": {
   "scifact": {
    "n": 300,
    "delta": 0.004802888875157289,
    "ci": [
     -0.008796397198375988,
     0.01907232288298107
    ]
   },
   "nfcorpus": {
    "n": 323,
    "delta": 0.010223726480966991,
    "ci": [
     0.0007607675600003981,
     0.019699609800694916
    ]
   },
   "fiqa": {
    "n": 300,
    "delta": -0.00845894752640707,
    "ci": [
     -0.021015969319745528,
     0.004047176725747511
    ]
   },
   "mean3": {
    "delta": 0.0021892226099057364,
    "ci": [
     -0.004674867516169616,
     0.00919203144124684
    ]
   }
  }
 },
 "means_check": {
  "v12_choice": {
   "scifact": 0.7791315841203013,
   "nfcorpus": 0.3574565625731375,
   "fiqa": 0.39568181370489885,
   "mean3": 0.5107566534661125
  },
  "v12_batch": {
   "scifact": 0.7665569870587879,
   "nfcorpus": 0.3438686030320072,
   "fiqa": 0.38369687156770177,
   "mean3": 0.49804082055283233
  },
  "flash_choice": {
   "scifact": 0.7713598759339452,
   "nfcorpus": 0.3540923295129741,
   "fiqa": 0.37523792404129463,
   "mean3": 0.5002300431627379
  },
  "rerank_mixedbread_v31": {
   "scifact": 0.7674914198991408,
   "nfcorpus": 0.3666336737446512,
   "fiqa": 0.39770443002598993,
   "mean3": 0.5106098412232607
  },
  "rerank_voyage3": {
   "scifact": 0.754657970730636,
   "nfcorpus": 0.35669798964188537,
   "fiqa": 0.40195928990676405,
   "mean3": 0.5044384167597619
  },
  "rerank_cohere": {
   "scifact": 0.7454041499445423,
   "nfcorpus": 0.33946734201298917,
   "fiqa": 0.37357333413687255,
   "mean3": 0.4861482753648014
  },
  "jev_batch_generic": {
   "scifact": 0.7683290081756673,
   "nfcorpus": 0.3582005994328261,
   "fiqa": 0.3760346534492147,
   "mean3": 0.5008547536859027
  },
  "jev_pair_generic": {
   "scifact": 0.7721709227158032,
   "nfcorpus": 0.3579923218604908,
   "fiqa": 0.37624320373016584,
   "mean3": 0.5021354827688199
  },
  "pplx_decider_choice_generic": {
   "scifact": 0.7760365238734583,
   "nfcorpus": 0.34678496424875166,
   "fiqa": 0.38153235857391093,
   "mean3": 0.5014512822320404
  },
  "pplx_decider_batch_generic": {
   "scifact": 0.7697435540120146,
   "nfcorpus": 0.3523659908017212,
   "fiqa": 0.37680526654713786,
   "mean3": 0.4996382704536246
  },
  "clef_flash_pair_generic": {
   "scifact": 0.7686173593041447,
   "nfcorpus": 0.3548353803128227,
   "fiqa": 0.37153230814287425,
   "mean3": 0.49832834925328057
  },
  "openai_decisions_luna_choice_generic": {
   "scifact": 0.7506458856327859,
   "nfcorpus": 0.3109481761526264,
   "fiqa": 0.3547108332322962,
   "mean3": 0.47210163167256947
  },
  "bm25": {
   "scifact": 0.6669140404875062,
   "nfcorpus": 0.3103988583346399,
   "fiqa": 0.23444863831592955,
   "mean3": 0.4039205123793586
  }
 },
 "p_at_1": {
  "v12_choice": {
   "scifact": 0.7,
   "nfcorpus": 0.5201238390092879,
   "fiqa": 0.4666666666666667
  },
  "v12_batch": {
   "scifact": 0.69,
   "nfcorpus": 0.4674922600619195,
   "fiqa": 0.4266666666666667
  },
  "flash_choice": {
   "scifact": 0.6933333333333334,
   "nfcorpus": 0.5201238390092879,
   "fiqa": 0.43666666666666665
  },
  "rerank_mixedbread_v31": {
   "scifact": 0.6766666666666666,
   "nfcorpus": 0.5201238390092879,
   "fiqa": 0.48333333333333334
  },
  "rerank_voyage3": {
   "scifact": 0.66,
   "nfcorpus": 0.5139318885448917,
   "fiqa": 0.49
  },
  "rerank_cohere": {
   "scifact": 0.66,
   "nfcorpus": 0.48297213622291024,
   "fiqa": 0.43
  },
  "jev_batch_generic": {
   "scifact": 0.6966666666666667,
   "nfcorpus": 0.5108359133126935,
   "fiqa": 0.44333333333333336
  },
  "jev_pair_generic": {
   "scifact": 0.71,
   "nfcorpus": 0.5232198142414861,
   "fiqa": 0.44
  },
  "pplx_decider_choice_generic": {
   "scifact": 0.7066666666666667,
   "nfcorpus": 0.5232198142414861,
   "fiqa": 0.4533333333333333
  },
  "pplx_decider_batch_generic": {
   "scifact": 0.7033333333333334,
   "nfcorpus": 0.4984520123839009,
   "fiqa": 0.44
  },
  "clef_flash_pair_generic": {
   "scifact": 0.6933333333333334,
   "nfcorpus": 0.5077399380804953,
   "fiqa": 0.4266666666666667
  },
  "openai_decisions_luna_choice_generic": {
   "scifact": 0.6766666666666666,
   "nfcorpus": 0.5139318885448917,
   "fiqa": 0.42
  },
  "bm25": {
   "scifact": 0.54,
   "nfcorpus": 0.43962848297213625,
   "fiqa": 0.2
  }
 },
 "shape_gap_change_v1_to_v12": {
  "scifact": {
   "n": 300,
   "did": 0.006281627200069293,
   "ci": [
    -0.007552180121077039,
    0.020084084419362692
   ]
  },
  "nfcorpus": {
   "n": 323,
   "did": 0.019168986094099557,
   "ci": [
    0.008528964924377429,
    0.029848768825929607
   ]
  },
  "fiqa": {
   "n": 300,
   "did": 0.0072578501104237384,
   "ci": [
    -0.002934724392480992,
    0.017452556825366228
   ]
  },
  "mean3": {
   "did": 0.010902821134864196,
   "ci": [
    0.00441981398151142,
    0.017744145856083037
   ]
  }
 },
 "spend": {
  "maximum_new_usd": 10,
  "accounted_usd": 1.3105449399999298,
  "received_usd": 0.68241154,
  "unresolved_reserved_usd": 0.6281334000000001,
  "attempts": 4733,
  "host": {
   "concurrency": 4,
   "request_starts_per_second": 5,
   "latency": "client dispatch through JSON response; excludes pacing; no historical same-host inference",
   "machine": "Linux server"
  },
  "updated_at": "2026-10-24T13:58:58.056911+00:00"
 },
 "v1_receipt_tokens": {
  "source": "rerank-bench 6b2f656 results/pplx/<ds>_test_pplx_decider_<shape>_generic.json.gz summary.input_tokens / n_queries; model pplx-decider-v1-27b, $0.04/M, Oct 3-4",
  "batch": {
   "scifact": 327045.0,
   "nfcorpus": 328711.1145510836,
   "fiqa": 213415.6
  },
  "choice": {
   "scifact": 11168.5,
   "nfcorpus": 11224.037151702787,
   "fiqa": 7380.8533333333335
  },
  "pair": {
   "scifact": 15641.873333333333,
   "nfcorpus": 15300.340557275542,
   "fiqa": 11674.083333333334
  }
 },
 "validation": "Scores, costs and paired intervals recomputed from the per-query run files with separate code; matches the eval to 4 decimal places. Not separately reviewed beyond that recompute.",
 "source_note": "Commit ids refer to a private benchmark repository. They identify the runs; the repository is not publicly available.",
 "price_notes": "usd_per_1k values match the original snapshot except mixedbread_v31, re-estimated on 2026-10-24 (see its price_basis; the snapshot value is kept as usd_per_1k_original_snapshot). Rows with price_basis.method == 'estimate' are not billed usage."
}