{ "protocol": { "version": "two-stage-retrospective-20261023-v1", "design": "Exploratory internal evaluation specified after inspecting the single-model and panel results; no new model calls", "sourceHashes": { "https://eternalsearch.net/omega/grok-generated-confirmation-20261023/report.json": "7c79ef7647a3ae8ba0a8917ab5ebfc509071e23da7993b57a61424029f180e29", "https://eternalsearch.net/omega/grok-generated-confirmation-20261023/report-strict.json": "3291d27d10a0a8631cd3964437856f552c08de5edb3f5596f726adf83c54e922" }, "firstStage": [ "gpt-6-astra", "claude-sonnet-5-5" ], "secondStage": "Every 3-, 4-, or 5-model subset of the five original predictors; retain at least one yes", "cohort": "Fixed all-five complete-case cohort within each response-handling condition; paired first-stage, panel-only, cascade and coin use identical identities", "endpoints": { "core": "At least two of three judges give article-correspondence score >=3", "strict": "At least two of three judges give article-correspondence score >=4" }, "uncertainty": "Descriptive pointwise 95% percentile intervals; paired original-task cluster bootstrap, 2000 draws, seed 20261022; no multiplicity adjustment, no independent validation", "workload": "One prediction unit per model and hypothesis, reusing first-stage votes in panels that contain that model; not HTTP calls, tokens, money or elapsed time", "controls": "Original recorded coin restricted to identical full-cohort or survivor identities; panel-only and first-stage-only controls", "invariants": [ "Final positives are a subset of gate positives", "An OR panel including the reused gate leaves its survivors unchanged" ] }, "conditions": [ { "handling": "extracted", "endpoint": "core", "totalOriginal": 250, "rows": [ { "task": "R42581103", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42581103", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42601407", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42601407", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42605193", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42608398", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42627902", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42630044", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H3", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42639736", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642518", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642518", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642521", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642521", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H2", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42647119", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H4", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42647635", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H4", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H5", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42663085", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H3", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42664348", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H3", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42665751", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H2", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H3", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42689491", "id": "H4", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42689491", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42696595", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42698274", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42723546", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42723546", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42744804", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42758848", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42758848", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42758848", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42758848", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42758848", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42759510", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775695", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775698", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775698", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42786310", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42805954", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H3", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42810392", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42811497", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42821663", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42821663", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42822442", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42826184", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42839094", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42841227", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42842666", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H3", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42842666", "id": "H4", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42844664", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42844664", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H5", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42853938", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42853938", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42858183", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42806945", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42808804", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42808804", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42585340", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42669067", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H4", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42811065", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42841610", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42806003", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42806572", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42748238", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42748238", "id": "H2", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42748238", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42748238", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42748238", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } } ], "results": [ { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 88, "matches": 33, "known": 40, "missed": 7, "counts": { "yes_true": 33, "yes_false": 55, "no_true": 7, "no_false": 145 }, "rates": { "ppv": 0.375, "npv": 0.9539473684210527, "sens": 0.825, "spec": 0.725, "youden": 0.5499999999999998, "accuracy": 0.7416666666666667, "balancedAccuracy": 0.7749999999999999 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 127, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 91, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.28346456692913385, "npv": 0.9646017699115044, "sens": 0.9, "spec": 0.545, "youden": 0.44500000000000006, "accuracy": 0.6041666666666666, "balancedAccuracy": 0.7225 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 126, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 90, "no_true": 4, "no_false": 110 }, "rates": { "ppv": 0.2857142857142857, "npv": 0.9649122807017544, "sens": 0.9, "spec": 0.55, "youden": 0.4500000000000002, "accuracy": 0.6083333333333333, "balancedAccuracy": 0.7250000000000001 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 47, "matches": 24, "known": 40, "missed": 16, "counts": { "yes_true": 24, "yes_false": 23, "no_true": 16, "no_false": 177 }, "rates": { "ppv": 0.5106382978723404, "npv": 0.917098445595855, "sens": 0.6, "spec": 0.885, "youden": 0.4849999999999999, "accuracy": 0.8375, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 47, "matches": 24, "known": 25, "missed": 1, "counts": { "yes_true": 24, "yes_false": 23, "no_true": 1, "no_false": 5 }, "rates": { "ppv": 0.5106382978723404, "npv": 0.8333333333333334, "sens": 0.96, "spec": 0.17857142857142858, "youden": 0.13857142857142857, "accuracy": 0.5471698113207547, "balancedAccuracy": 0.5692857142857143 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 1, "removedAtSecond": 6, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 720, "saved": 321 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [ "R42844664|H5" ], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 47, "matches": 24, "known": 40, "missed": 16, "counts": { "yes_true": 24, "yes_false": 23, "no_true": 16, "no_false": 177 }, "rates": { "ppv": 0.5106382978723404, "npv": 0.917098445595855, "sens": 0.6, "spec": 0.885, "youden": 0.4849999999999999, "accuracy": 0.8375, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.33177650925445423, "npv": 0.07094459944200882, "sens": 0.04999999999999993, "spec": 0.39, "youden": 0.43999999999999995, "accuracy": 0.33333333333333337, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.34146341463414637, "hi": 0.6666666666666666, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.19597063203186713, "hi": 0.47287352365380886, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8666577540106952, "hi": 0.9585545643929276, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.013355963513792686, "hi": 0.13653157380309291, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4473684210526316, "hi": 0.7608954451345755, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.1777777777777778, "hi": 0.32142857142857145, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8443396226415094, "hi": 0.9218851198186528, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3190316275764037, "hi": 0.46429473304473295, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.326411311991571, "hi": 0.6601307189542482, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.1871005187714375, "hi": 0.7250182319802436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7916666666666666, "hi": 0.8792708333333328, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.25, "hi": 0.4126041666666661, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6632056559957855, "hi": 0.8300653594771241, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09355025938571875, "hi": 0.3625091159901218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 128, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 92, "no_true": 4, "no_false": 108 }, "rates": { "ppv": 0.28125, "npv": 0.9642857142857143, "sens": 0.9, "spec": 0.54, "youden": 0.43999999999999995, "accuracy": 0.6, "balancedAccuracy": 0.72 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 132, "matches": 37, "known": 40, "missed": 3, "counts": { "yes_true": 37, "yes_false": 95, "no_true": 3, "no_false": 105 }, "rates": { "ppv": 0.2803030303030303, "npv": 0.9722222222222222, "sens": 0.925, "spec": 0.525, "youden": 0.4500000000000002, "accuracy": 0.5916666666666667, "balancedAccuracy": 0.7250000000000001 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 960, "saved": 561 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 87, "matches": 31, "known": 40, "missed": 9, "counts": { "yes_true": 31, "yes_false": 56, "no_true": 9, "no_false": 144 }, "rates": { "ppv": 0.3563218390804598, "npv": 0.9411764705882353, "sens": 0.775, "spec": 0.72, "youden": 0.4950000000000001, "accuracy": 0.7291666666666666, "balancedAccuracy": 0.7475 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 82, "matches": 31, "known": 40, "missed": 9, "counts": { "yes_true": 31, "yes_false": 51, "no_true": 9, "no_false": 149 }, "rates": { "ppv": 0.3780487804878049, "npv": 0.9430379746835443, "sens": 0.775, "spec": 0.745, "youden": 0.52, "accuracy": 0.75, "balancedAccuracy": 0.76 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 41, "matches": 23, "known": 40, "missed": 17, "counts": { "yes_true": 23, "yes_false": 18, "no_true": 17, "no_false": 182 }, "rates": { "ppv": 0.5609756097560976, "npv": 0.914572864321608, "sens": 0.575, "spec": 0.91, "youden": 0.4849999999999999, "accuracy": 0.8541666666666666, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 41, "matches": 23, "known": 25, "missed": 2, "counts": { "yes_true": 23, "yes_false": 18, "no_true": 2, "no_false": 10 }, "rates": { "ppv": 0.5609756097560976, "npv": 0.8333333333333334, "sens": 0.92, "spec": 0.35714285714285715, "youden": 0.27714285714285714, "accuracy": 0.6226415094339622, "balancedAccuracy": 0.6385714285714286 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 2, "removedAtSecond": 12, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 720, "saved": 321 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42664348|H2", "R42665751|H3", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [ "R42647635|H4", "R42647635|H5" ], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 41, "matches": 23, "known": 40, "missed": 17, "counts": { "yes_true": 23, "yes_false": 18, "no_true": 17, "no_false": 182 }, "rates": { "ppv": 0.5609756097560976, "npv": 0.914572864321608, "sens": 0.575, "spec": 0.91, "youden": 0.4849999999999999, "accuracy": 0.8541666666666666, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.38211382113821146, "npv": 0.06841901816776186, "sens": 0.02499999999999991, "spec": 0.41500000000000004, "youden": 0.43999999999999995, "accuracy": 0.35, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.4, "hi": 0.7105409356725145, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.24599704833407451, "hi": 0.5088831126264505, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.862732593037215, "hi": 0.958974358974359, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014031472062003645, "hi": 0.12814408433828076, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.43997560975609756, "hi": 0.7317073170731707, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.18181818181818182, "hi": 0.2857142857142857, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8768472906403941, "hi": 0.9414893617021277, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3471502590673575, "hi": 0.48223350253807107, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33767411658109336, "hi": 0.6513397665828855, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.20419519809857586, "hi": 0.7186122095181655, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.8083333333333333, "hi": 0.8958333333333334, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2666666666666667, "hi": 0.4291666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6688370582905466, "hi": 0.8256698832914428, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.10209759904928793, "hi": 0.35930610475908276, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 87, "matches": 32, "known": 40, "missed": 8, "counts": { "yes_true": 32, "yes_false": 55, "no_true": 8, "no_false": 145 }, "rates": { "ppv": 0.367816091954023, "npv": 0.9477124183006536, "sens": 0.8, "spec": 0.725, "youden": 0.5249999999999999, "accuracy": 0.7375, "balancedAccuracy": 0.7625 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 94, "matches": 33, "known": 40, "missed": 7, "counts": { "yes_true": 33, "yes_false": 61, "no_true": 7, "no_false": 139 }, "rates": { "ppv": 0.35106382978723405, "npv": 0.952054794520548, "sens": 0.825, "spec": 0.695, "youden": 0.52, "accuracy": 0.7166666666666667, "balancedAccuracy": 0.76 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 960, "saved": 561 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 124, "matches": 34, "known": 40, "missed": 6, "counts": { "yes_true": 34, "yes_false": 90, "no_true": 6, "no_false": 110 }, "rates": { "ppv": 0.27419354838709675, "npv": 0.9482758620689655, "sens": 0.85, "spec": 0.55, "youden": 0.3999999999999999, "accuracy": 0.6, "balancedAccuracy": 0.7 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 23, "known": 40, "missed": 17, "counts": { "yes_true": 23, "yes_false": 23, "no_true": 17, "no_false": 177 }, "rates": { "ppv": 0.5, "npv": 0.9123711340206185, "sens": 0.575, "spec": 0.885, "youden": 0.45999999999999996, "accuracy": 0.8333333333333334, "balancedAccuracy": 0.73 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 46, "matches": 23, "known": 25, "missed": 2, "counts": { "yes_true": 23, "yes_false": 23, "no_true": 2, "no_false": 5 }, "rates": { "ppv": 0.5, "npv": 0.7142857142857143, "sens": 0.92, "spec": 0.17857142857142858, "youden": 0.09857142857142853, "accuracy": 0.5283018867924528, "balancedAccuracy": 0.5492857142857143 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 2, "removedAtSecond": 7, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 720, "saved": 321 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [ "R42822442|H5", "R42841227|H5" ], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 23, "known": 40, "missed": 17, "counts": { "yes_true": 23, "yes_false": 23, "no_true": 17, "no_false": 177 }, "rates": { "ppv": 0.5, "npv": 0.9123711340206185, "sens": 0.575, "spec": 0.885, "youden": 0.45999999999999996, "accuracy": 0.8333333333333334, "balancedAccuracy": 0.73 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.32113821138211385, "npv": 0.06621728786677239, "sens": 0.02499999999999991, "spec": 0.39, "youden": 0.41500000000000004, "accuracy": 0.3291666666666667, "balancedAccuracy": 0.20750000000000002 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.32650688182249643, "hi": 0.6591029980657639, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.17680695475433011, "hi": 0.45896640913294096, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8622404996481351, "hi": 0.9545454545454546, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.00415158962113274, "hi": 0.13435502211945075, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.41456241032998564, "hi": 0.7381211180124223, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2222789115646259, "hi": 0.3077586206896548, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8442164953206399, "hi": 0.9238674819258575, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3142857142857143, "hi": 0.4673366834170854, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.29177497207101544, "hi": 0.6306756594393262, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.1535312133079911, "hi": 0.7141594922959327, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7875, "hi": 0.875, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.24572916666666658, "hi": 0.41250000000000003, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6458874860355077, "hi": 0.8153378297196631, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07676560665399555, "hi": 0.35707974614796634, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 127, "matches": 35, "known": 40, "missed": 5, "counts": { "yes_true": 35, "yes_false": 92, "no_true": 5, "no_false": 108 }, "rates": { "ppv": 0.2755905511811024, "npv": 0.9557522123893806, "sens": 0.875, "spec": 0.54, "youden": 0.41500000000000004, "accuracy": 0.5958333333333333, "balancedAccuracy": 0.7075 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 131, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 95, "no_true": 4, "no_false": 105 }, "rates": { "ppv": 0.2748091603053435, "npv": 0.963302752293578, "sens": 0.9, "spec": 0.525, "youden": 0.42500000000000004, "accuracy": 0.5875, "balancedAccuracy": 0.7125 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 960, "saved": 561 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 126, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 90, "no_true": 4, "no_false": 110 }, "rates": { "ppv": 0.2857142857142857, "npv": 0.9649122807017544, "sens": 0.9, "spec": 0.55, "youden": 0.4500000000000002, "accuracy": 0.6083333333333333, "balancedAccuracy": 0.7250000000000001 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 49, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 24, "no_true": 15, "no_false": 176 }, "rates": { "ppv": 0.5102040816326531, "npv": 0.9214659685863874, "sens": 0.625, "spec": 0.88, "youden": 0.5049999999999999, "accuracy": 0.8375, "balancedAccuracy": 0.7525 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 49, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 24, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.5102040816326531, "npv": 1, "sens": 1, "spec": 0.14285714285714285, "youden": 0.1428571428571428, "accuracy": 0.5471698113207547, "balancedAccuracy": 0.5714285714285714 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 720, "saved": 321 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 49, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 24, "no_true": 15, "no_false": 176 }, "rates": { "ppv": 0.5102040816326531, "npv": 0.9214659685863874, "sens": 0.625, "spec": 0.88, "youden": 0.5049999999999999, "accuracy": 0.8375, "balancedAccuracy": 0.7525 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.3313422930147669, "npv": 0.07531212243254126, "sens": 0.07499999999999996, "spec": 0.385, "youden": 0.45999999999999996, "accuracy": 0.33333333333333337, "balancedAccuracy": 0.22999999999999998 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.34782608695652173, "hi": 0.66, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.19999821810406274, "hi": 0.460235504531655, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.870461419969192, "hi": 0.9629629629629629, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.015827412422861835, "hi": 0.14372819720483065, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.839622641509434, "hi": 0.9166774611398962, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3128125, "hi": 0.45877706802150636, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.34133829852579844, "hi": 0.6756741206157227, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19970782670286855, "hi": 0.7500000000000001, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7916666666666666, "hi": 0.8791666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.24583333333333335, "hi": 0.41666666666666663, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6706691492628992, "hi": 0.8378370603078613, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09985391335143427, "hi": 0.37500000000000006, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 129, "matches": 37, "known": 40, "missed": 3, "counts": { "yes_true": 37, "yes_false": 92, "no_true": 3, "no_false": 108 }, "rates": { "ppv": 0.2868217054263566, "npv": 0.972972972972973, "sens": 0.925, "spec": 0.54, "youden": 0.4650000000000001, "accuracy": 0.6041666666666666, "balancedAccuracy": 0.7325 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 49, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 24, "no_true": 15, "no_false": 176 }, "rates": { "ppv": 0.5102040816326531, "npv": 0.9214659685863874, "sens": 0.625, "spec": 0.88, "youden": 0.5049999999999999, "accuracy": 0.8375, "balancedAccuracy": 0.7525 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 49, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 24, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.5102040816326531, "npv": 1, "sens": 1, "spec": 0.14285714285714285, "youden": 0.1428571428571428, "accuracy": 0.5471698113207547, "balancedAccuracy": 0.5714285714285714 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 240, "survivors": 53, "additionalModels": 4, "second": 212, "total": 452, "fullPanel": 960, "saved": 508 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 49, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 24, "no_true": 15, "no_false": 176 }, "rates": { "ppv": 0.5102040816326531, "npv": 0.9214659685863874, "sens": 0.625, "spec": 0.88, "youden": 0.5049999999999999, "accuracy": 0.8375, "balancedAccuracy": 0.7525 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.3313422930147669, "npv": 0.07531212243254126, "sens": 0.07499999999999996, "spec": 0.385, "youden": 0.45999999999999996, "accuracy": 0.33333333333333337, "balancedAccuracy": 0.22999999999999998 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.34782608695652173, "hi": 0.66, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.19999821810406274, "hi": 0.460235504531655, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.870461419969192, "hi": 0.9629629629629629, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.015827412422861835, "hi": 0.14372819720483065, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.839622641509434, "hi": 0.9166774611398962, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3128125, "hi": 0.45877706802150636, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.34133829852579844, "hi": 0.6756741206157227, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19970782670286855, "hi": 0.7500000000000001, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7916666666666666, "hi": 0.8791666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.24583333333333335, "hi": 0.41666666666666663, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6706691492628992, "hi": 0.8378370603078613, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09985391335143427, "hi": 0.37500000000000006, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 130, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 94, "no_true": 4, "no_false": 106 }, "rates": { "ppv": 0.27692307692307694, "npv": 0.9636363636363636, "sens": 0.9, "spec": 0.53, "youden": 0.43000000000000016, "accuracy": 0.5916666666666667, "balancedAccuracy": 0.7150000000000001 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 960, "saved": 561 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 133, "matches": 37, "known": 40, "missed": 3, "counts": { "yes_true": 37, "yes_false": 96, "no_true": 3, "no_false": 104 }, "rates": { "ppv": 0.2781954887218045, "npv": 0.9719626168224299, "sens": 0.925, "spec": 0.52, "youden": 0.44500000000000006, "accuracy": 0.5875, "balancedAccuracy": 0.7225 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4716981132075472, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 13 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.5, "sens": 0.48, "spec": 0.4642857142857143, "youden": -0.055714285714285716, "accuracy": 0.4716981132075472, "balancedAccuracy": 0.47214285714285714 } }, "lostAtFirst": 15, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 4, "second": 212, "total": 452, "fullPanel": 1200, "saved": 748 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2", "R42748238|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 25, "known": 40, "missed": 15, "counts": { "yes_true": 25, "yes_false": 28, "no_true": 15, "no_false": 172 }, "rates": { "ppv": 0.4716981132075472, "npv": 0.9197860962566845, "sens": 0.625, "spec": 0.86, "youden": 0.4849999999999999, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.7424999999999999 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.292836324589661, "npv": 0.07363225010283836, "sens": 0.07499999999999996, "spec": 0.365, "youden": 0.43999999999999995, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.21999999999999997 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3125, "hi": 0.6271186440677966, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16257990715781684, "hi": 0.42648835789932144, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8673937639590925, "hi": 0.9623655913978495, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.01343976693934248, "hi": 0.14231947603531508, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.47058823529411764, "hi": 0.7857829670329667, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 0.35559523809523785, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8130772063901294, "hi": 0.900990099009901, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2885548220436279, "hi": 0.4422110552763819, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31714011362120614, "hi": 0.6595507338190258, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.17787472943722948, "hi": 0.7354577229038436, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6585700568106031, "hi": 0.8297753669095129, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08893736471861474, "hi": 0.3677288614519218, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 88, "matches": 33, "known": 40, "missed": 7, "counts": { "yes_true": 33, "yes_false": 55, "no_true": 7, "no_false": 145 }, "rates": { "ppv": 0.375, "npv": 0.9539473684210527, "sens": 0.825, "spec": 0.725, "youden": 0.5499999999999998, "accuracy": 0.7416666666666667, "balancedAccuracy": 0.7749999999999999 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 127, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 91, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.28346456692913385, "npv": 0.9646017699115044, "sens": 0.9, "spec": 0.545, "youden": 0.44500000000000006, "accuracy": 0.6041666666666666, "balancedAccuracy": 0.7225 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 126, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 90, "no_true": 4, "no_false": 110 }, "rates": { "ppv": 0.2857142857142857, "npv": 0.9649122807017544, "sens": 0.9, "spec": 0.55, "youden": 0.4500000000000002, "accuracy": 0.6083333333333333, "balancedAccuracy": 0.7250000000000001 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 128, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 92, "no_true": 4, "no_false": 108 }, "rates": { "ppv": 0.28125, "npv": 0.9642857142857143, "sens": 0.9, "spec": 0.54, "youden": 0.43999999999999995, "accuracy": 0.6, "balancedAccuracy": 0.72 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 42, "matches": 20, "known": 40, "missed": 20, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 20, "no_false": 178 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.898989898989899, "sens": 0.5, "spec": 0.89, "youden": 0.3900000000000001, "accuracy": 0.825, "balancedAccuracy": 0.6950000000000001 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 42, "matches": 20, "known": 21, "missed": 1, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 1, "no_false": 3 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.75, "sens": 0.9523809523809523, "spec": 0.12, "youden": 0.07238095238095221, "accuracy": 0.5, "balancedAccuracy": 0.5361904761904761 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 1, "removedAtSecond": 4, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 720, "saved": 342 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42826184|H1", "R42839094|H4", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [ "R42669067|H5" ], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 42, "matches": 20, "known": 40, "missed": 20, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 20, "no_false": 178 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.898989898989899, "sens": 0.5, "spec": 0.89, "youden": 0.3900000000000001, "accuracy": 0.825, "balancedAccuracy": 0.6950000000000001 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.29732868757259, "npv": 0.05283605283605286, "sens": -0.050000000000000044, "spec": 0.395, "youden": 0.3450000000000002, "accuracy": 0.3208333333333333, "balancedAccuracy": 0.1725000000000001 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.30952380952380953, "hi": 0.625, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1594681280883401, "hi": 0.4268553386637842, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8477157360406091, "hi": 0.9444444444444444, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.005842779849915821, "hi": 0.11357785469529713, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.375, "hi": 0.6458639705882352, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.27503989361702125, "hi": 0.21742527173913026, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8613861386138614, "hi": 0.9195979899497487, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3207502031235895, "hi": 0.4718090452261306, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.25305423949656525, "hi": 0.5451162989919381, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.09641891166101973, "hi": 0.6286050828098225, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7833333333333333, "hi": 0.8625, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23749999999999993, "hi": 0.4000000000000001, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6265271197482827, "hi": 0.7725581494959691, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.04820945583050987, "hi": 0.31430254140491126, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 132, "matches": 37, "known": 40, "missed": 3, "counts": { "yes_true": 37, "yes_false": 95, "no_true": 3, "no_false": 105 }, "rates": { "ppv": 0.2803030303030303, "npv": 0.9722222222222222, "sens": 0.925, "spec": 0.525, "youden": 0.4500000000000002, "accuracy": 0.5916666666666667, "balancedAccuracy": 0.7250000000000001 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 960, "saved": 582 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 87, "matches": 31, "known": 40, "missed": 9, "counts": { "yes_true": 31, "yes_false": 56, "no_true": 9, "no_false": 144 }, "rates": { "ppv": 0.3563218390804598, "npv": 0.9411764705882353, "sens": 0.775, "spec": 0.72, "youden": 0.4950000000000001, "accuracy": 0.7291666666666666, "balancedAccuracy": 0.7475 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 82, "matches": 31, "known": 40, "missed": 9, "counts": { "yes_true": 31, "yes_false": 51, "no_true": 9, "no_false": 149 }, "rates": { "ppv": 0.3780487804878049, "npv": 0.9430379746835443, "sens": 0.775, "spec": 0.745, "youden": 0.52, "accuracy": 0.75, "balancedAccuracy": 0.76 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 87, "matches": 32, "known": 40, "missed": 8, "counts": { "yes_true": 32, "yes_false": 55, "no_true": 8, "no_false": 145 }, "rates": { "ppv": 0.367816091954023, "npv": 0.9477124183006536, "sens": 0.8, "spec": 0.725, "youden": 0.5249999999999999, "accuracy": 0.7375, "balancedAccuracy": 0.7625 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 39, "matches": 20, "known": 40, "missed": 20, "counts": { "yes_true": 20, "yes_false": 19, "no_true": 20, "no_false": 181 }, "rates": { "ppv": 0.5128205128205128, "npv": 0.900497512437811, "sens": 0.5, "spec": 0.905, "youden": 0.405, "accuracy": 0.8375, "balancedAccuracy": 0.7025 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 39, "matches": 20, "known": 21, "missed": 1, "counts": { "yes_true": 20, "yes_false": 19, "no_true": 1, "no_false": 6 }, "rates": { "ppv": 0.5128205128205128, "npv": 0.8571428571428571, "sens": 0.9523809523809523, "spec": 0.24, "youden": 0.19238095238095232, "accuracy": 0.5652173913043478, "balancedAccuracy": 0.5961904761904762 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 1, "removedAtSecond": 7, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 720, "saved": 342 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42826184|H1", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [ "R42669067|H5" ], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 39, "matches": 20, "known": 40, "missed": 20, "counts": { "yes_true": 20, "yes_false": 19, "no_true": 20, "no_false": 181 }, "rates": { "ppv": 0.5128205128205128, "npv": 0.900497512437811, "sens": 0.5, "spec": 0.905, "youden": 0.405, "accuracy": 0.8375, "balancedAccuracy": 0.7025 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.3339587242026266, "npv": 0.05434366628396481, "sens": -0.050000000000000044, "spec": 0.41000000000000003, "youden": 0.3600000000000001, "accuracy": 0.33333333333333337, "balancedAccuracy": 0.18000000000000005 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.34285714285714286, "hi": 0.6666666666666666, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.19032465784249908, "hi": 0.4702726833976832, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8497285651053194, "hi": 0.9458128078817734, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.005185134121191673, "hi": 0.1152189998157726, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.375, "hi": 0.6458639705882352, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.27503989361702125, "hi": 0.21742527173913026, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8743712015575259, "hi": 0.9350080845771145, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3350484634735856, "hi": 0.48022137119372305, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2652070895522389, "hi": 0.560404820462371, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.11024593925423146, "hi": 0.6424961316366228, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7916666666666666, "hi": 0.8791666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.25, "hi": 0.41666666666666663, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6326035447761195, "hi": 0.7802024102311854, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05512296962711573, "hi": 0.3212480658183114, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 94, "matches": 33, "known": 40, "missed": 7, "counts": { "yes_true": 33, "yes_false": 61, "no_true": 7, "no_false": 139 }, "rates": { "ppv": 0.35106382978723405, "npv": 0.952054794520548, "sens": 0.825, "spec": 0.695, "youden": 0.52, "accuracy": 0.7166666666666667, "balancedAccuracy": 0.76 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 960, "saved": 582 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 124, "matches": 34, "known": 40, "missed": 6, "counts": { "yes_true": 34, "yes_false": 90, "no_true": 6, "no_false": 110 }, "rates": { "ppv": 0.27419354838709675, "npv": 0.9482758620689655, "sens": 0.85, "spec": 0.55, "youden": 0.3999999999999999, "accuracy": 0.6, "balancedAccuracy": 0.7 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 127, "matches": 35, "known": 40, "missed": 5, "counts": { "yes_true": 35, "yes_false": 92, "no_true": 5, "no_false": 108 }, "rates": { "ppv": 0.2755905511811024, "npv": 0.9557522123893806, "sens": 0.875, "spec": 0.54, "youden": 0.41500000000000004, "accuracy": 0.5958333333333333, "balancedAccuracy": 0.7075 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 42, "matches": 20, "known": 40, "missed": 20, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 20, "no_false": 178 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.898989898989899, "sens": 0.5, "spec": 0.89, "youden": 0.3900000000000001, "accuracy": 0.825, "balancedAccuracy": 0.6950000000000001 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 42, "matches": 20, "known": 21, "missed": 1, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 1, "no_false": 3 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.75, "sens": 0.9523809523809523, "spec": 0.12, "youden": 0.07238095238095221, "accuracy": 0.5, "balancedAccuracy": 0.5361904761904761 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 1, "removedAtSecond": 4, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 720, "saved": 342 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [ "R42669067|H5" ], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 42, "matches": 20, "known": 40, "missed": 20, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 20, "no_false": 178 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.898989898989899, "sens": 0.5, "spec": 0.89, "youden": 0.3900000000000001, "accuracy": 0.825, "balancedAccuracy": 0.6950000000000001 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.29732868757259, "npv": 0.05283605283605286, "sens": -0.050000000000000044, "spec": 0.395, "youden": 0.3450000000000002, "accuracy": 0.3208333333333333, "balancedAccuracy": 0.1725000000000001 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3157894736842105, "hi": 0.627906976744186, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1595238550591595, "hi": 0.4328048410226082, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8469387755102041, "hi": 0.9447236180904522, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.006611558537526979, "hi": 0.1135215242366079, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.375, "hi": 0.6458639705882352, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.27503989361702125, "hi": 0.21742527173913026, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8609792963433952, "hi": 0.92, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3214072112098428, "hi": 0.4712041884816754, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2509624837417273, "hi": 0.5480088221519139, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.09373221402490477, "hi": 0.6286154916512056, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7791666666666667, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23749999999999993, "hi": 0.4041666666666666, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6254812418708636, "hi": 0.774004411075957, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.046866107012452383, "hi": 0.3143077458256028, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 131, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 95, "no_true": 4, "no_false": 105 }, "rates": { "ppv": 0.2748091603053435, "npv": 0.963302752293578, "sens": 0.9, "spec": 0.525, "youden": 0.42500000000000004, "accuracy": 0.5875, "balancedAccuracy": 0.7125 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 960, "saved": 582 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 126, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 90, "no_true": 4, "no_false": 110 }, "rates": { "ppv": 0.2857142857142857, "npv": 0.9649122807017544, "sens": 0.9, "spec": 0.55, "youden": 0.4500000000000002, "accuracy": 0.6083333333333333, "balancedAccuracy": 0.7250000000000001 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 43, "matches": 20, "known": 40, "missed": 20, "counts": { "yes_true": 20, "yes_false": 23, "no_true": 20, "no_false": 177 }, "rates": { "ppv": 0.46511627906976744, "npv": 0.8984771573604061, "sens": 0.5, "spec": 0.885, "youden": 0.385, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.6925 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 43, "matches": 20, "known": 21, "missed": 1, "counts": { "yes_true": 20, "yes_false": 23, "no_true": 1, "no_false": 2 }, "rates": { "ppv": 0.46511627906976744, "npv": 0.6666666666666666, "sens": 0.9523809523809523, "spec": 0.08, "youden": 0.0323809523809524, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.5161904761904762 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 1, "removedAtSecond": 3, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 720, "saved": 342 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [ "R42669067|H5" ], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 43, "matches": 20, "known": 40, "missed": 20, "counts": { "yes_true": 20, "yes_false": 23, "no_true": 20, "no_false": 177 }, "rates": { "ppv": 0.46511627906976744, "npv": 0.8984771573604061, "sens": 0.5, "spec": 0.885, "youden": 0.385, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.6925 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.2862544904518812, "npv": 0.052323311206559975, "sens": -0.050000000000000044, "spec": 0.39, "youden": 0.3400000000000001, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.17000000000000004 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.30434782608695654, "hi": 0.6136363636363636, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14981561253275588, "hi": 0.4151121463422252, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8469387755102041, "hi": 0.9441694867456288, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.007125285991195288, "hi": 0.11309584432792213, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.375, "hi": 0.6458639705882352, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.27503989361702125, "hi": 0.21742527173913026, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8557680348258706, "hi": 0.9137055837563451, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3155274044669759, "hi": 0.4656862745098039, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.24690461314172657, "hi": 0.5419129262131686, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.0892145701053041, "hi": 0.6247302566406334, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7791666666666667, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.39593749999999944, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6234523065708633, "hi": 0.7709564631065843, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.04460728505265205, "hi": 0.3123651283203167, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 129, "matches": 37, "known": 40, "missed": 3, "counts": { "yes_true": 37, "yes_false": 92, "no_true": 3, "no_false": 108 }, "rates": { "ppv": 0.2868217054263566, "npv": 0.972972972972973, "sens": 0.925, "spec": 0.54, "youden": 0.4650000000000001, "accuracy": 0.6041666666666666, "balancedAccuracy": 0.7325 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 960, "saved": 582 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 130, "matches": 36, "known": 40, "missed": 4, "counts": { "yes_true": 36, "yes_false": 94, "no_true": 4, "no_false": 106 }, "rates": { "ppv": 0.27692307692307694, "npv": 0.9636363636363636, "sens": 0.9, "spec": 0.53, "youden": 0.43000000000000016, "accuracy": 0.5916666666666667, "balancedAccuracy": 0.7150000000000001 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 43, "matches": 20, "known": 40, "missed": 20, "counts": { "yes_true": 20, "yes_false": 23, "no_true": 20, "no_false": 177 }, "rates": { "ppv": 0.46511627906976744, "npv": 0.8984771573604061, "sens": 0.5, "spec": 0.885, "youden": 0.385, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.6925 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 43, "matches": 20, "known": 21, "missed": 1, "counts": { "yes_true": 20, "yes_false": 23, "no_true": 1, "no_false": 2 }, "rates": { "ppv": 0.46511627906976744, "npv": 0.6666666666666666, "sens": 0.9523809523809523, "spec": 0.08, "youden": 0.0323809523809524, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.5161904761904762 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 1, "removedAtSecond": 3, "workload": { "first": 240, "survivors": 46, "additionalModels": 4, "second": 184, "total": 424, "fullPanel": 960, "saved": 536 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [ "R42669067|H5" ], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 43, "matches": 20, "known": 40, "missed": 20, "counts": { "yes_true": 20, "yes_false": 23, "no_true": 20, "no_false": 177 }, "rates": { "ppv": 0.46511627906976744, "npv": 0.8984771573604061, "sens": 0.5, "spec": 0.885, "youden": 0.385, "accuracy": 0.8208333333333333, "balancedAccuracy": 0.6925 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.2862544904518812, "npv": 0.052323311206559975, "sens": -0.050000000000000044, "spec": 0.39, "youden": 0.3400000000000001, "accuracy": 0.31666666666666665, "balancedAccuracy": 0.17000000000000004 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.30434782608695654, "hi": 0.6136363636363636, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14981561253275588, "hi": 0.4151121463422252, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8469387755102041, "hi": 0.9441694867456288, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.007125285991195288, "hi": 0.11309584432792213, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.375, "hi": 0.6458639705882352, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.27503989361702125, "hi": 0.21742527173913026, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8557680348258706, "hi": 0.9137055837563451, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3155274044669759, "hi": 0.4656862745098039, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.24690461314172657, "hi": 0.5419129262131686, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.0892145701053041, "hi": 0.6247302566406334, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7791666666666667, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23333333333333328, "hi": 0.39593749999999944, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6234523065708633, "hi": 0.7709564631065843, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.04460728505265205, "hi": 0.3123651283203167, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 133, "matches": 37, "known": 40, "missed": 3, "counts": { "yes_true": 37, "yes_false": 96, "no_true": 3, "no_false": 104 }, "rates": { "ppv": 0.2781954887218045, "npv": 0.9719626168224299, "sens": 0.925, "spec": 0.52, "youden": 0.44500000000000006, "accuracy": 0.5875, "balancedAccuracy": 0.7225 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.45652173913043476, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 12 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5217391304347826, "sens": 0.47619047619047616, "spec": 0.48, "youden": -0.043809523809523854, "accuracy": 0.4782608695652174, "balancedAccuracy": 0.4780952380952381 } }, "lostAtFirst": 19, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 4, "second": 184, "total": 424, "fullPanel": 1200, "saved": 776 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42822442|H5", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 21, "known": 40, "missed": 19, "counts": { "yes_true": 21, "yes_false": 25, "no_true": 19, "no_false": 175 }, "rates": { "ppv": 0.45652173913043476, "npv": 0.9020618556701031, "sens": 0.525, "spec": 0.875, "youden": 0.3999999999999999, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.7 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 22, "known": 40, "missed": 18, "counts": { "yes_true": 22, "yes_false": 101, "no_true": 18, "no_false": 99 }, "rates": { "ppv": 0.17886178861788618, "npv": 0.8461538461538461, "sens": 0.55, "spec": 0.495, "youden": 0.04499999999999993, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.5225 } }, "differences": { "ppv": 0.27765995051254855, "npv": 0.05590800951625696, "sens": -0.025000000000000022, "spec": 0.38, "youden": 0.355, "accuracy": 0.3125, "balancedAccuracy": 0.1775 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3022674418604651, "hi": 0.5918367346938775, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.09917355371900827, "hi": 0.2644628099173554, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.14461111111111113, "hi": 0.4045429814221665, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8504961006356497, "hi": 0.9479166666666666, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7796385180740963, "hi": 0.9083333333333333, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.002009089882565423, "hi": 0.11502647920261547, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.3958333333333333, "hi": 0.6764705882352942, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.3870967741935484, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2368734335839599, "hi": 0.22727272727272724, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8446576789576941, "hi": 0.9045345477386935, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4307673267326733, "hi": 0.5598133971291867, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30612244897959184, "hi": 0.45544554455445546, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2499703726160291, "hi": 0.5596035382467276, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13679319002769436, "hi": 0.21395911418368543, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10209446936024082, "hi": 0.6353985086315581, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7708333333333334, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.44166666666666665, "hi": 0.5666666666666667, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22499999999999998, "hi": 0.39583333333333326, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6249851863080146, "hi": 0.7798017691233637, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4316034049861528, "hi": 0.6069795570918427, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05104723468012041, "hi": 0.31769925431577906, "validReplicates": 2000, "degenerate": false } } } } } ] }, { "handling": "extracted", "endpoint": "strict", "totalOriginal": 250, "rows": [ { "task": "R42581103", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42581103", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42601407", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42601407", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42605193", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42608398", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42627902", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42630044", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42639736", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642518", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642518", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642521", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642521", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42647119", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42647635", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42663085", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42664348", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42665751", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42689491", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42689491", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42696595", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42698274", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42723546", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42723546", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42744804", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42758848", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42758848", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42758848", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42758848", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42758848", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42759510", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775695", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775698", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775698", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42786310", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42805954", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H3", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42810392", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42811497", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42821663", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42821663", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42822442", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42826184", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42839094", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42841227", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42842666", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42842666", "id": "H4", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42844664", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42844664", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42853938", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42853938", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42858183", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42806945", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42808804", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42808804", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42585340", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42669067", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42811065", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42841610", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42806003", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42806572", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42748238", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42748238", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42748238", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42748238", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42748238", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } } ], "results": [ { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 88, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 82, "no_true": 1, "no_false": 151 }, "rates": { "ppv": 0.06818181818181818, "npv": 0.993421052631579, "sens": 0.8571428571428571, "spec": 0.648068669527897, "youden": 0.5052115266707542, "accuracy": 0.6541666666666667, "balancedAccuracy": 0.7526057633353771 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 127, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 120, "no_true": 0, "no_false": 113 }, "rates": { "ppv": 0.05511811023622047, "npv": 1, "sens": 1, "spec": 0.48497854077253216, "youden": 0.4849785407725322, "accuracy": 0.5, "balancedAccuracy": 0.7424892703862661 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 126, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 119, "no_true": 0, "no_false": 114 }, "rates": { "ppv": 0.05555555555555555, "npv": 1, "sens": 1, "spec": 0.4892703862660944, "youden": 0.48927038626609454, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.7446351931330473 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 47, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 41, "no_true": 1, "no_false": 192 }, "rates": { "ppv": 0.1276595744680851, "npv": 0.9948186528497409, "sens": 0.8571428571428571, "spec": 0.8240343347639485, "youden": 0.6811771919068055, "accuracy": 0.825, "balancedAccuracy": 0.8405885959534027 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 47, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 41, "no_true": 0, "no_false": 6 }, "rates": { "ppv": 0.1276595744680851, "npv": 1, "sens": 1, "spec": 0.1276595744680851, "youden": 0.12765957446808507, "accuracy": 0.22641509433962265, "balancedAccuracy": 0.5638297872340425 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 6, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 720, "saved": 321 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 47, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 41, "no_true": 1, "no_false": 192 }, "rates": { "ppv": 0.1276595744680851, "npv": 0.9948186528497409, "sens": 0.8571428571428571, "spec": 0.8240343347639485, "youden": 0.6811771919068055, "accuracy": 0.825, "balancedAccuracy": 0.8405885959534027 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10326933056564608, "npv": 0.029006687037775092, "sens": 0.42857142857142855, "spec": 0.33905579399141633, "youden": 0.7676272225628448, "accuracy": 0.3416666666666666, "balancedAccuracy": 0.3838136112814224 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.22916666666666666, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.02779267329650075, "hi": 0.19791666666666666, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9840425531914894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00004108412585095694, "hi": 0.06667226812304528, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7869565217391304, "hi": 0.8614718614718615, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.26723407364114554, "hi": 0.4085152636447733, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3460326438267616, "hi": 0.8540772532188843, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.13784261715296184, "hi": 1.34342818064793, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7875, "hi": 0.8625, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26250000000000007, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6730163219133808, "hi": 0.9270386266094421, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06892130857648092, "hi": 0.671714090323965, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 128, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 121, "no_true": 0, "no_false": 112 }, "rates": { "ppv": 0.0546875, "npv": 1, "sens": 1, "spec": 0.48068669527896996, "youden": 0.4806866952789699, "accuracy": 0.49583333333333335, "balancedAccuracy": 0.740343347639485 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 132, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 125, "no_true": 0, "no_false": 108 }, "rates": { "ppv": 0.05303030303030303, "npv": 1, "sens": 1, "spec": 0.463519313304721, "youden": 0.46351931330472107, "accuracy": 0.4791666666666667, "balancedAccuracy": 0.7317596566523605 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 960, "saved": 561 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 87, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 81, "no_true": 1, "no_false": 152 }, "rates": { "ppv": 0.06896551724137931, "npv": 0.9934640522875817, "sens": 0.8571428571428571, "spec": 0.6523605150214592, "youden": 0.5095033721643163, "accuracy": 0.6583333333333333, "balancedAccuracy": 0.7547516860821581 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 82, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 76, "no_true": 1, "no_false": 157 }, "rates": { "ppv": 0.07317073170731707, "npv": 0.9936708860759493, "sens": 0.8571428571428571, "spec": 0.6738197424892703, "youden": 0.5309625996321274, "accuracy": 0.6791666666666667, "balancedAccuracy": 0.7654812998160637 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 41, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 35, "no_true": 1, "no_false": 198 }, "rates": { "ppv": 0.14634146341463414, "npv": 0.9949748743718593, "sens": 0.8571428571428571, "spec": 0.8497854077253219, "youden": 0.706928264868179, "accuracy": 0.85, "balancedAccuracy": 0.8534641324340895 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 41, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 35, "no_true": 0, "no_false": 12 }, "rates": { "ppv": 0.14634146341463414, "npv": 1, "sens": 1, "spec": 0.2553191489361702, "youden": 0.25531914893617014, "accuracy": 0.33962264150943394, "balancedAccuracy": 0.6276595744680851 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 12, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 720, "saved": 321 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42664348|H2", "R42665751|H3", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 41, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 35, "no_true": 1, "no_false": 198 }, "rates": { "ppv": 0.14634146341463414, "npv": 0.9949748743718593, "sens": 0.8571428571428571, "spec": 0.8497854077253219, "youden": 0.706928264868179, "accuracy": 0.85, "balancedAccuracy": 0.8534641324340895 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.12195121951219512, "npv": 0.029162908559893452, "sens": 0.42857142857142855, "spec": 0.3648068669527897, "youden": 0.7933782955242182, "accuracy": 0.36666666666666664, "balancedAccuracy": 0.3966891477621091 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.05125, "hi": 0.2682926829268293, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.03498993984062711, "hi": 0.2339722015884806, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9845360824742269, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0, "hi": 0.06677445866505985, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8162393162393162, "hi": 0.8836331767056385, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2999894514767932, "hi": 0.42918454935622313, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3643198609300303, "hi": 0.8777292576419216, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.16662834661761686, "hi": 1.3688438639776874, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8166666666666667, "hi": 0.8833333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2958333333333334, "hi": 0.4375, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6821599304650152, "hi": 0.9388646288209608, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08331417330880843, "hi": 0.6844219319888437, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 87, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 81, "no_true": 1, "no_false": 152 }, "rates": { "ppv": 0.06896551724137931, "npv": 0.9934640522875817, "sens": 0.8571428571428571, "spec": 0.6523605150214592, "youden": 0.5095033721643163, "accuracy": 0.6583333333333333, "balancedAccuracy": 0.7547516860821581 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 94, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 88, "no_true": 1, "no_false": 145 }, "rates": { "ppv": 0.06382978723404255, "npv": 0.9931506849315068, "sens": 0.8571428571428571, "spec": 0.6223175965665236, "youden": 0.47946045370938073, "accuracy": 0.6291666666666667, "balancedAccuracy": 0.7397302268546904 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 960, "saved": 561 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 124, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 117, "no_true": 0, "no_false": 116 }, "rates": { "ppv": 0.056451612903225805, "npv": 1, "sens": 1, "spec": 0.4978540772532189, "youden": 0.49785407725321895, "accuracy": 0.5125, "balancedAccuracy": 0.7489270386266095 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 7 }, "rates": { "ppv": 0.13043478260869565, "npv": 1, "sens": 1, "spec": 0.14893617021276595, "youden": 0.14893617021276606, "accuracy": 0.24528301886792453, "balancedAccuracy": 0.574468085106383 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 7, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 720, "saved": 321 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028636755401461288, "hi": 0.2030093102992084, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9841269841269841, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00015485395713306924, "hi": 0.06667091174944574, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7887931034482759, "hi": 0.8680851063829788, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2675362976406533, "hi": 0.41880341880341876, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3570520581113802, "hi": 0.860176430316039, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14875733675715325, "hi": 1.3477754237288138, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7875, "hi": 0.8708333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2666666666666666, "hi": 0.425, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6785260290556901, "hi": 0.9300882151580194, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07437866837857662, "hi": 0.6738877118644069, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 127, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 120, "no_true": 0, "no_false": 113 }, "rates": { "ppv": 0.05511811023622047, "npv": 1, "sens": 1, "spec": 0.48497854077253216, "youden": 0.4849785407725322, "accuracy": 0.5, "balancedAccuracy": 0.7424892703862661 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 2, "second": 106, "total": 346, "fullPanel": 720, "saved": 374 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 131, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 124, "no_true": 0, "no_false": 109 }, "rates": { "ppv": 0.05343511450381679, "npv": 1, "sens": 1, "spec": 0.4678111587982833, "youden": 0.46781115879828317, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.7339055793991416 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 960, "saved": 561 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 126, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 119, "no_true": 0, "no_false": 114 }, "rates": { "ppv": 0.05555555555555555, "npv": 1, "sens": 1, "spec": 0.4892703862660944, "youden": 0.48927038626609454, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.7446351931330473 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 49, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 43, "no_true": 1, "no_false": 190 }, "rates": { "ppv": 0.12244897959183673, "npv": 0.9947643979057592, "sens": 0.8571428571428571, "spec": 0.8154506437768241, "youden": 0.6725935009196813, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.8362967504598406 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 49, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 43, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.12244897959183673, "npv": 1, "sens": 1, "spec": 0.0851063829787234, "youden": 0.0851063829787233, "accuracy": 0.18867924528301888, "balancedAccuracy": 0.5425531914893617 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 720, "saved": 321 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 49, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 43, "no_true": 1, "no_false": 190 }, "rates": { "ppv": 0.12244897959183673, "npv": 0.9947643979057592, "sens": 0.8571428571428571, "spec": 0.8154506437768241, "youden": 0.6725935009196813, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.8362967504598406 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.09805873568939771, "npv": 0.02895243209379339, "sens": 0.42857142857142855, "spec": 0.3304721030042919, "youden": 0.7590435315757206, "accuracy": 0.3333333333333333, "balancedAccuracy": 0.3795217657878603 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0425531914893617, "hi": 0.22, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.02559456196596319, "hi": 0.18917420694060033, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9838709677419355, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0002489554190709869, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7777777777777778, "hi": 0.853448275862069, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2564102564102565, "hi": 0.4016745342059149, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33887983485441114, "hi": 0.8461538461538463, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.1324363520640116, "hi": 1.3347457627118644, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7791666666666667, "hi": 0.8541666666666666, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.25416666666666665, "hi": 0.4125, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694399174272055, "hi": 0.9230769230769231, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.0662181760320058, "hi": 0.6673728813559322, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 129, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 122, "no_true": 0, "no_false": 111 }, "rates": { "ppv": 0.05426356589147287, "npv": 1, "sens": 1, "spec": 0.47639484978540775, "youden": 0.4763948497854078, "accuracy": 0.49166666666666664, "balancedAccuracy": 0.7381974248927039 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 49, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 43, "no_true": 1, "no_false": 190 }, "rates": { "ppv": 0.12244897959183673, "npv": 0.9947643979057592, "sens": 0.8571428571428571, "spec": 0.8154506437768241, "youden": 0.6725935009196813, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.8362967504598406 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 49, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 43, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.12244897959183673, "npv": 1, "sens": 1, "spec": 0.0851063829787234, "youden": 0.0851063829787233, "accuracy": 0.18867924528301888, "balancedAccuracy": 0.5425531914893617 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 240, "survivors": 53, "additionalModels": 4, "second": 212, "total": 452, "fullPanel": 960, "saved": 508 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 49, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 43, "no_true": 1, "no_false": 190 }, "rates": { "ppv": 0.12244897959183673, "npv": 0.9947643979057592, "sens": 0.8571428571428571, "spec": 0.8154506437768241, "youden": 0.6725935009196813, "accuracy": 0.8166666666666667, "balancedAccuracy": 0.8362967504598406 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.09805873568939771, "npv": 0.02895243209379339, "sens": 0.42857142857142855, "spec": 0.3304721030042919, "youden": 0.7590435315757206, "accuracy": 0.3333333333333333, "balancedAccuracy": 0.3795217657878603 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0425531914893617, "hi": 0.22, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.02559456196596319, "hi": 0.18917420694060033, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9838709677419355, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0002489554190709869, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7777777777777778, "hi": 0.853448275862069, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2564102564102565, "hi": 0.4016745342059149, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33887983485441114, "hi": 0.8461538461538463, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.1324363520640116, "hi": 1.3347457627118644, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7791666666666667, "hi": 0.8541666666666666, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.25416666666666665, "hi": 0.4125, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694399174272055, "hi": 0.9230769230769231, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.0662181760320058, "hi": 0.6673728813559322, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 130, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 123, "no_true": 0, "no_false": 110 }, "rates": { "ppv": 0.05384615384615385, "npv": 1, "sens": 1, "spec": 0.4721030042918455, "youden": 0.4721030042918455, "accuracy": 0.4875, "balancedAccuracy": 0.7360515021459227 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 3, "second": 159, "total": 399, "fullPanel": 960, "saved": 561 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 133, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 126, "no_true": 0, "no_false": 107 }, "rates": { "ppv": 0.05263157894736842, "npv": 1, "sens": 1, "spec": 0.4592274678111588, "youden": 0.45922746781115875, "accuracy": 0.475, "balancedAccuracy": 0.7296137339055794 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "conditional": { "hypotheses": 53, "questions": 42, "selected": 53, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11320754716981132, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11320754716981132, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 53, "questions": 42, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 22 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.8461538461538461, "sens": 0.3333333333333333, "spec": 0.46808510638297873, "youden": -0.198581560283688, "accuracy": 0.4528301886792453, "balancedAccuracy": 0.400709219858156 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 53, "additionalModels": 4, "second": 212, "total": 452, "fullPanel": 1200, "saved": 748 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42758848|H5", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 53, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 47, "no_true": 1, "no_false": 186 }, "rates": { "ppv": 0.11320754716981132, "npv": 0.9946524064171123, "sens": 0.8571428571428571, "spec": 0.7982832618025751, "youden": 0.6554261189454322, "accuracy": 0.8, "balancedAccuracy": 0.8277130594727161 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.0888173032673723, "npv": 0.028840440605146456, "sens": 0.42857142857142855, "spec": 0.313304721030043, "youden": 0.7418761496014715, "accuracy": 0.3166666666666667, "balancedAccuracy": 0.37093807480073576 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.0392156862745098, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021407274413388543, "hi": 0.17388145607013525, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9835164835164835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0006287384474191913, "hi": 0.06666739319965125, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7574468085106383, "hi": 0.8369098712446352, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.23931623931623935, "hi": 0.38626609442060084, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3176798131247284, "hi": 0.8297872340425532, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.11468176815847379, "hi": 1.3193759458737648, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7583333333333333, "hi": 0.8375, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23739583333333328, "hi": 0.3958333333333333, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6588399065623642, "hi": 0.9148936170212766, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.057340884079236897, "hi": 0.6596879729368824, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 88, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 82, "no_true": 1, "no_false": 151 }, "rates": { "ppv": 0.06818181818181818, "npv": 0.993421052631579, "sens": 0.8571428571428571, "spec": 0.648068669527897, "youden": 0.5052115266707542, "accuracy": 0.6541666666666667, "balancedAccuracy": 0.7526057633353771 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 127, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 120, "no_true": 0, "no_false": 113 }, "rates": { "ppv": 0.05511811023622047, "npv": 1, "sens": 1, "spec": 0.48497854077253216, "youden": 0.4849785407725322, "accuracy": 0.5, "balancedAccuracy": 0.7424892703862661 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 126, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 119, "no_true": 0, "no_false": 114 }, "rates": { "ppv": 0.05555555555555555, "npv": 1, "sens": 1, "spec": 0.4892703862660944, "youden": 0.48927038626609454, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.7446351931330473 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 128, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 121, "no_true": 0, "no_false": 112 }, "rates": { "ppv": 0.0546875, "npv": 1, "sens": 1, "spec": 0.48068669527896996, "youden": 0.4806866952789699, "accuracy": 0.49583333333333335, "balancedAccuracy": 0.740343347639485 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 42, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 1, "no_false": 197 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.9949494949494949, "sens": 0.8571428571428571, "spec": 0.8454935622317596, "youden": 0.7026364193746168, "accuracy": 0.8458333333333333, "balancedAccuracy": 0.8513182096873084 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 42, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.14285714285714285, "npv": 1, "sens": 1, "spec": 0.1, "youden": 0.10000000000000009, "accuracy": 0.21739130434782608, "balancedAccuracy": 0.55 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 720, "saved": 342 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42826184|H1", "R42839094|H4", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 42, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 1, "no_false": 197 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.9949494949494949, "sens": 0.8571428571428571, "spec": 0.8454935622317596, "youden": 0.7026364193746168, "accuracy": 0.8458333333333333, "balancedAccuracy": 0.8513182096873084 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.11846689895470383, "npv": 0.029137529137529095, "sens": 0.42857142857142855, "spec": 0.3605150214592275, "youden": 0.7890864500306561, "accuracy": 0.3625, "balancedAccuracy": 0.39454322501532807 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.04878048780487805, "hi": 0.25, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.03339080754252921, "hi": 0.2160564411786246, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9845360824742269, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0, "hi": 0.06667226812304528, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8217391304347826, "hi": 0.8701298701298701, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.28812104459154725, "hi": 0.4316384180790959, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3504273504273505, "hi": 0.8663793103448276, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.16639610389610376, "hi": 1.3644067796610169, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8208333333333333, "hi": 0.8708333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.28333333333333344, "hi": 0.43749999999999994, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6752136752136753, "hi": 0.9331896551724138, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08319805194805188, "hi": 0.6822033898305084, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 132, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 125, "no_true": 0, "no_false": 108 }, "rates": { "ppv": 0.05303030303030303, "npv": 1, "sens": 1, "spec": 0.463519313304721, "youden": 0.46351931330472107, "accuracy": 0.4791666666666667, "balancedAccuracy": 0.7317596566523605 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 960, "saved": 582 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 87, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 81, "no_true": 1, "no_false": 152 }, "rates": { "ppv": 0.06896551724137931, "npv": 0.9934640522875817, "sens": 0.8571428571428571, "spec": 0.6523605150214592, "youden": 0.5095033721643163, "accuracy": 0.6583333333333333, "balancedAccuracy": 0.7547516860821581 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 82, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 76, "no_true": 1, "no_false": 157 }, "rates": { "ppv": 0.07317073170731707, "npv": 0.9936708860759493, "sens": 0.8571428571428571, "spec": 0.6738197424892703, "youden": 0.5309625996321274, "accuracy": 0.6791666666666667, "balancedAccuracy": 0.7654812998160637 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 87, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 81, "no_true": 1, "no_false": 152 }, "rates": { "ppv": 0.06896551724137931, "npv": 0.9934640522875817, "sens": 0.8571428571428571, "spec": 0.6523605150214592, "youden": 0.5095033721643163, "accuracy": 0.6583333333333333, "balancedAccuracy": 0.7547516860821581 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 39, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 33, "no_true": 1, "no_false": 200 }, "rates": { "ppv": 0.15384615384615385, "npv": 0.9950248756218906, "sens": 0.8571428571428571, "spec": 0.8583690987124464, "youden": 0.7155119558553036, "accuracy": 0.8583333333333333, "balancedAccuracy": 0.8577559779276518 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 39, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 33, "no_true": 0, "no_false": 7 }, "rates": { "ppv": 0.15384615384615385, "npv": 1, "sens": 1, "spec": 0.175, "youden": 0.17500000000000004, "accuracy": 0.2826086956521739, "balancedAccuracy": 0.5875 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 7, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 720, "saved": 342 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42826184|H1", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 39, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 33, "no_true": 1, "no_false": 200 }, "rates": { "ppv": 0.15384615384615385, "npv": 0.9950248756218906, "sens": 0.8571428571428571, "spec": 0.8583690987124464, "youden": 0.7155119558553036, "accuracy": 0.8583333333333333, "balancedAccuracy": 0.8577559779276518 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.12945590994371484, "npv": 0.029212909809924748, "sens": 0.42857142857142855, "spec": 0.3733905579399142, "youden": 0.8019619865113429, "accuracy": 0.37499999999999994, "balancedAccuracy": 0.40098099325567144 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.05405405405405406, "hi": 0.268342122610415, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.03827920662598082, "hi": 0.23461829836829834, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9847715736040609, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0, "hi": 0.06697340571118887, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8333333333333334, "hi": 0.8851063829787233, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.29957805907172996, "hi": 0.44206008583690987, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.36728777343184127, "hi": 0.8803418803418803, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.17521367521367545, "hi": 1.3745475356854298, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8291666666666667, "hi": 0.8875, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.29999999999999993, "hi": 0.44583333333333336, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6836438867159206, "hi": 0.9401709401709402, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08760683760683773, "hi": 0.6872737678427149, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 94, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 88, "no_true": 1, "no_false": 145 }, "rates": { "ppv": 0.06382978723404255, "npv": 0.9931506849315068, "sens": 0.8571428571428571, "spec": 0.6223175965665236, "youden": 0.47946045370938073, "accuracy": 0.6291666666666667, "balancedAccuracy": 0.7397302268546904 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 960, "saved": 582 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 124, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 117, "no_true": 0, "no_false": 116 }, "rates": { "ppv": 0.056451612903225805, "npv": 1, "sens": 1, "spec": 0.4978540772532189, "youden": 0.49785407725321895, "accuracy": 0.5125, "balancedAccuracy": 0.7489270386266095 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 2, "second": 92, "total": 332, "fullPanel": 720, "saved": 388 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 127, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 120, "no_true": 0, "no_false": 113 }, "rates": { "ppv": 0.05511811023622047, "npv": 1, "sens": 1, "spec": 0.48497854077253216, "youden": 0.4849785407725322, "accuracy": 0.5, "balancedAccuracy": 0.7424892703862661 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 42, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 1, "no_false": 197 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.9949494949494949, "sens": 0.8571428571428571, "spec": 0.8454935622317596, "youden": 0.7026364193746168, "accuracy": 0.8458333333333333, "balancedAccuracy": 0.8513182096873084 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 42, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.14285714285714285, "npv": 1, "sens": 1, "spec": 0.1, "youden": 0.10000000000000009, "accuracy": 0.21739130434782608, "balancedAccuracy": 0.55 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 720, "saved": 342 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 42, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 1, "no_false": 197 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.9949494949494949, "sens": 0.8571428571428571, "spec": 0.8454935622317596, "youden": 0.7026364193746168, "accuracy": 0.8458333333333333, "balancedAccuracy": 0.8513182096873084 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.11846689895470383, "npv": 0.029137529137529095, "sens": 0.42857142857142855, "spec": 0.3605150214592275, "youden": 0.7890864500306561, "accuracy": 0.3625, "balancedAccuracy": 0.39454322501532807 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.05, "hi": 0.25, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.033050847457627125, "hi": 0.2140713860403838, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9845360824742269, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0, "hi": 0.06672328853120851, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8220338983050848, "hi": 0.8717948717948718, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2900401069518716, "hi": 0.4317329703941672, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.35159984789222076, "hi": 0.8658008658008658, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.16239316239316245, "hi": 1.3628975981824083, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8208333333333333, "hi": 0.8708333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2875000000000001, "hi": 0.4334374999999995, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6757999239461103, "hi": 0.9329004329004329, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08119658119658123, "hi": 0.6814487990912041, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 131, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 124, "no_true": 0, "no_false": 109 }, "rates": { "ppv": 0.05343511450381679, "npv": 1, "sens": 1, "spec": 0.4678111587982833, "youden": 0.46781115879828317, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.7339055793991416 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 960, "saved": 582 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 126, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 119, "no_true": 0, "no_false": 114 }, "rates": { "ppv": 0.05555555555555555, "npv": 1, "sens": 1, "spec": 0.4892703862660944, "youden": 0.48927038626609454, "accuracy": 0.5041666666666667, "balancedAccuracy": 0.7446351931330473 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 43, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 37, "no_true": 1, "no_false": 196 }, "rates": { "ppv": 0.13953488372093023, "npv": 0.9949238578680203, "sens": 0.8571428571428571, "spec": 0.8412017167381974, "youden": 0.6983445738810545, "accuracy": 0.8416666666666667, "balancedAccuracy": 0.8491722869405273 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 43, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 37, "no_true": 0, "no_false": 3 }, "rates": { "ppv": 0.13953488372093023, "npv": 1, "sens": 1, "spec": 0.075, "youden": 0.07499999999999996, "accuracy": 0.1956521739130435, "balancedAccuracy": 0.5375 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 3, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 720, "saved": 342 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 43, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 37, "no_true": 1, "no_false": 196 }, "rates": { "ppv": 0.13953488372093023, "npv": 0.9949238578680203, "sens": 0.8571428571428571, "spec": 0.8412017167381974, "youden": 0.6983445738810545, "accuracy": 0.8416666666666667, "balancedAccuracy": 0.8491722869405273 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.11514463981849121, "npv": 0.029111892056054467, "sens": 0.42857142857142855, "spec": 0.3562231759656653, "youden": 0.7847946045370938, "accuracy": 0.35833333333333334, "balancedAccuracy": 0.3923973022685469 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.047619047619047616, "hi": 0.2444557823129251, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.03197020436946999, "hi": 0.20952935638479964, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9845360824742269, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0, "hi": 0.06667226812304528, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.81817218798151, "hi": 0.8658008658008658, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.28389691732662276, "hi": 0.4267388538660185, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.35023225566703836, "hi": 0.8607677490368741, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.16207264957264947, "hi": 1.3601694915254237, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8166666666666667, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2833333333333333, "hi": 0.42927083333333277, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6751161278335192, "hi": 0.9303838745184371, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08103632478632473, "hi": 0.6800847457627118, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 129, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 122, "no_true": 0, "no_false": 111 }, "rates": { "ppv": 0.05426356589147287, "npv": 1, "sens": 1, "spec": 0.47639484978540775, "youden": 0.4763948497854078, "accuracy": 0.49166666666666664, "balancedAccuracy": 0.7381974248927039 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 3, "second": 138, "total": 378, "fullPanel": 960, "saved": 582 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 130, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 123, "no_true": 0, "no_false": 110 }, "rates": { "ppv": 0.05384615384615385, "npv": 1, "sens": 1, "spec": 0.4721030042918455, "youden": 0.4721030042918455, "accuracy": 0.4875, "balancedAccuracy": 0.7360515021459227 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 43, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 37, "no_true": 1, "no_false": 196 }, "rates": { "ppv": 0.13953488372093023, "npv": 0.9949238578680203, "sens": 0.8571428571428571, "spec": 0.8412017167381974, "youden": 0.6983445738810545, "accuracy": 0.8416666666666667, "balancedAccuracy": 0.8491722869405273 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 43, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 37, "no_true": 0, "no_false": 3 }, "rates": { "ppv": 0.13953488372093023, "npv": 1, "sens": 1, "spec": 0.075, "youden": 0.07499999999999996, "accuracy": 0.1956521739130435, "balancedAccuracy": 0.5375 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 3, "workload": { "first": 240, "survivors": 46, "additionalModels": 4, "second": 184, "total": 424, "fullPanel": 960, "saved": 536 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 43, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 37, "no_true": 1, "no_false": 196 }, "rates": { "ppv": 0.13953488372093023, "npv": 0.9949238578680203, "sens": 0.8571428571428571, "spec": 0.8412017167381974, "youden": 0.6983445738810545, "accuracy": 0.8416666666666667, "balancedAccuracy": 0.8491722869405273 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.11514463981849121, "npv": 0.029111892056054467, "sens": 0.42857142857142855, "spec": 0.3562231759656653, "youden": 0.7847946045370938, "accuracy": 0.35833333333333334, "balancedAccuracy": 0.3923973022685469 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.047619047619047616, "hi": 0.2444557823129251, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.03197020436946999, "hi": 0.20952935638479964, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9845360824742269, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0, "hi": 0.06667226812304528, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.81817218798151, "hi": 0.8658008658008658, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.28389691732662276, "hi": 0.4267388538660185, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.35023225566703836, "hi": 0.8607677490368741, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.16207264957264947, "hi": 1.3601694915254237, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8166666666666667, "hi": 0.8666666666666667, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2833333333333333, "hi": 0.42927083333333277, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6751161278335192, "hi": 0.9303838745184371, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.08103632478632473, "hi": 0.6800847457627118, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "panelOnly": { "hypotheses": 240, "questions": 48, "selected": 133, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 126, "no_true": 0, "no_false": 107 }, "rates": { "ppv": 0.05263157894736842, "npv": 1, "sens": 1, "spec": 0.4592274678111588, "youden": 0.45922746781115875, "accuracy": 0.475, "balancedAccuracy": 0.7296137339055794 } }, "final": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "conditional": { "hypotheses": 46, "questions": 42, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13043478260869565, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13043478260869565, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "conditionalCoin": { "hypotheses": 46, "questions": 42, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 19 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8260869565217391, "sens": 0.3333333333333333, "spec": 0.475, "youden": -0.19166666666666665, "accuracy": 0.45652173913043476, "balancedAccuracy": 0.4041666666666667 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 240, "survivors": 46, "additionalModels": 4, "second": 184, "total": 424, "fullPanel": 1200, "saved": 776 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42758848|H5", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H4", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2", "R42748238|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 240, "questions": 48, "left": { "hypotheses": 240, "questions": 48, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 193 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9948453608247423, "sens": 0.8571428571428571, "spec": 0.8283261802575107, "youden": 0.6854690374003678, "accuracy": 0.8291666666666667, "balancedAccuracy": 0.8427345187001839 } }, "right": { "hypotheses": 240, "questions": 48, "selected": 123, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 120, "no_true": 4, "no_false": 113 }, "rates": { "ppv": 0.024390243902439025, "npv": 0.9658119658119658, "sens": 0.42857142857142855, "spec": 0.48497854077253216, "youden": -0.08645003065603929, "accuracy": 0.48333333333333334, "balancedAccuracy": 0.45677498467198036 } }, "differences": { "ppv": 0.10604453870625663, "npv": 0.029033395012776464, "sens": 0.42857142857142855, "spec": 0.34334763948497854, "youden": 0.7719190680564071, "accuracy": 0.3458333333333334, "balancedAccuracy": 0.38595953402820354 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23404255319148937, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.055566666666666605, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028092885375494068, "hi": 0.1979260149572649, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9842105263157894, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9285714285714286, "hi": 0.991869918699187, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.00008006172115664883, "hi": 0.06666952666952664, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.16666666666666663, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8008658008658008, "hi": 0.8540772532188842, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4242295839753467, "hi": 0.5474241004497751, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27541586485086217, "hi": 0.4112616365934949, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33880158437730296, "hi": 0.8491379310344827, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5235396924727338, "hi": 0.31499454446262964, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.14957264957264949, "hi": 1.3433298773891094, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8, "hi": 0.8583333333333333, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.42083333333333334, "hi": 0.55, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2749999999999999, "hi": 0.4166666666666667, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6694007921886515, "hi": 0.9245689655172413, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23823015376363313, "hi": 0.6574972722313148, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.07478632478632474, "hi": 0.6716649386945547, "validReplicates": 1998, "degenerate": false } } } } } ] }, { "handling": "strict", "endpoint": "core", "totalOriginal": 250, "rows": [ { "task": "R42581103", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42581103", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42601407", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42601407", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42605193", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42608398", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42627902", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42630044", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H3", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42639736", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42639736", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642518", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642518", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642521", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642521", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H2", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42647119", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H4", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42647635", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H4", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H5", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42663085", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H3", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42664348", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H3", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42665751", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H2", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H3", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42689491", "id": "H4", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42689491", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42696595", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42698274", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42723546", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42723546", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42744804", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775695", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775698", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775698", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42786310", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42805954", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H3", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42810392", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42811497", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42821663", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42821663", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42822442", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42826184", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42839094", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42841227", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42842666", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H3", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42842666", "id": "H4", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42844664", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42844664", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H5", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42853938", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42853938", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42858183", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42806945", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42808804", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42808804", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42585340", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42669067", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H4", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42811065", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42841610", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42806003", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42806572", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } } ], "results": [ { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 85, "matches": 32, "known": 39, "missed": 7, "counts": { "yes_true": 32, "yes_false": 53, "no_true": 7, "no_false": 138 }, "rates": { "ppv": 0.3764705882352941, "npv": 0.9517241379310345, "sens": 0.8205128205128205, "spec": 0.7225130890052356, "youden": 0.5430259095180561, "accuracy": 0.7391304347826086, "balancedAccuracy": 0.771512954759028 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 89, "no_true": 4, "no_false": 102 }, "rates": { "ppv": 0.28225806451612906, "npv": 0.9622641509433962, "sens": 0.8974358974358975, "spec": 0.5340314136125655, "youden": 0.43146731104846303, "accuracy": 0.5956521739130435, "balancedAccuracy": 0.7157336555242315 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 87, "no_true": 4, "no_false": 104 }, "rates": { "ppv": 0.28688524590163933, "npv": 0.9629629629629629, "sens": 0.8974358974358975, "spec": 0.5445026178010471, "youden": 0.4419385152369446, "accuracy": 0.6043478260869565, "balancedAccuracy": 0.7209692576184723 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 46, "matches": 24, "known": 39, "missed": 15, "counts": { "yes_true": 24, "yes_false": 22, "no_true": 15, "no_false": 169 }, "rates": { "ppv": 0.5217391304347826, "npv": 0.9184782608695652, "sens": 0.6153846153846154, "spec": 0.8848167539267016, "youden": 0.500201369311317, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.7501006846556585 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 46, "matches": 24, "known": 25, "missed": 1, "counts": { "yes_true": 24, "yes_false": 22, "no_true": 1, "no_false": 5 }, "rates": { "ppv": 0.5217391304347826, "npv": 0.8333333333333334, "sens": 0.96, "spec": 0.18518518518518517, "youden": 0.14518518518518508, "accuracy": 0.5576923076923077, "balancedAccuracy": 0.5725925925925925 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 1, "removedAtSecond": 6, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 690, "saved": 304 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [ "R42844664|H5" ], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 46, "matches": 24, "known": 39, "missed": 15, "counts": { "yes_true": 24, "yes_false": 22, "no_true": 15, "no_false": 169 }, "rates": { "ppv": 0.5217391304347826, "npv": 0.9184782608695652, "sens": 0.6153846153846154, "spec": 0.8848167539267016, "youden": 0.500201369311317, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.7501006846556585 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3422519509476031, "npv": 0.0777702962677953, "sens": 0.07692307692307698, "spec": 0.38743455497382195, "youden": 0.46435763189689894, "accuracy": 0.33478260869565224, "balancedAccuracy": 0.23217881594844947 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3478039041703638, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.19881636631153882, "hi": 0.4828079366114938, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8677168289046742, "hi": 0.9615437158469945, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.012664302690209456, "hi": 0.14266137673217305, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4531063988095238, "hi": 0.7777777777777778, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.15629111842105262, "hi": 0.33333333333333337, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8407960199004975, "hi": 0.9255417651694247, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3137106357694593, "hi": 0.46667512690355323, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.32271189047838794, "hi": 0.669309947299077, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.20323368999271063, "hi": 0.7431537828947369, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7869565217391304, "hi": 0.8869565217391304, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.25217391304347825, "hi": 0.4217391304347826, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6613559452391939, "hi": 0.8346549736495386, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.10161684499635532, "hi": 0.37157689144736844, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 89, "no_true": 4, "no_false": 102 }, "rates": { "ppv": 0.28225806451612906, "npv": 0.9622641509433962, "sens": 0.8974358974358975, "spec": 0.5340314136125655, "youden": 0.43146731104846303, "accuracy": 0.5956521739130435, "balancedAccuracy": 0.7157336555242315 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 128, "matches": 36, "known": 39, "missed": 3, "counts": { "yes_true": 36, "yes_false": 92, "no_true": 3, "no_false": 99 }, "rates": { "ppv": 0.28125, "npv": 0.9705882352941176, "sens": 0.9230769230769231, "spec": 0.518324607329843, "youden": 0.441401530406766, "accuracy": 0.5869565217391305, "balancedAccuracy": 0.720700765203383 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 920, "saved": 534 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 85, "matches": 30, "known": 39, "missed": 9, "counts": { "yes_true": 30, "yes_false": 55, "no_true": 9, "no_false": 136 }, "rates": { "ppv": 0.35294117647058826, "npv": 0.9379310344827586, "sens": 0.7692307692307693, "spec": 0.7120418848167539, "youden": 0.4812726540475232, "accuracy": 0.7217391304347827, "balancedAccuracy": 0.7406363270237616 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 79, "matches": 29, "known": 39, "missed": 10, "counts": { "yes_true": 29, "yes_false": 50, "no_true": 10, "no_false": 141 }, "rates": { "ppv": 0.3670886075949367, "npv": 0.9337748344370861, "sens": 0.7435897435897436, "spec": 0.7382198952879581, "youden": 0.48180963887770156, "accuracy": 0.7391304347826086, "balancedAccuracy": 0.7409048194388508 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 40, "matches": 22, "known": 39, "missed": 17, "counts": { "yes_true": 22, "yes_false": 18, "no_true": 17, "no_false": 173 }, "rates": { "ppv": 0.55, "npv": 0.9105263157894737, "sens": 0.5641025641025641, "spec": 0.9057591623036649, "youden": 0.46986172640622903, "accuracy": 0.8478260869565217, "balancedAccuracy": 0.7349308632031145 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 40, "matches": 22, "known": 25, "missed": 3, "counts": { "yes_true": 22, "yes_false": 18, "no_true": 3, "no_false": 9 }, "rates": { "ppv": 0.55, "npv": 0.75, "sens": 0.88, "spec": 0.3333333333333333, "youden": 0.21333333333333337, "accuracy": 0.5961538461538461, "balancedAccuracy": 0.6066666666666667 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 3, "removedAtSecond": 12, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 690, "saved": 304 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42664348|H2", "R42665751|H3", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [ "R42647635|H4", "R42647635|H5", "R42806572|H5" ], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 40, "matches": 22, "known": 39, "missed": 17, "counts": { "yes_true": 22, "yes_false": 18, "no_true": 17, "no_false": 173 }, "rates": { "ppv": 0.55, "npv": 0.9105263157894737, "sens": 0.5641025641025641, "spec": 0.9057591623036649, "youden": 0.46986172640622903, "accuracy": 0.8478260869565217, "balancedAccuracy": 0.7349308632031145 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.37051282051282053, "npv": 0.06981835118770385, "sens": 0.02564102564102566, "spec": 0.4083769633507853, "youden": 0.434017988991811, "accuracy": 0.34347826086956523, "balancedAccuracy": 0.2170089944959055 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.375, "hi": 0.7222222222222222, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.22708582696723267, "hi": 0.5212409211102588, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8571428571428571, "hi": 0.9574468085106383, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.013493483174671845, "hi": 0.1298713204653799, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.42849544072948326, "hi": 0.72, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.18004545454545454, "hi": 0.28003124999999984, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8645720108695653, "hi": 0.9433040654668297, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.336678653933173, "hi": 0.4842105263157894, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3220846102465134, "hi": 0.6385560214413488, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.20660448491970226, "hi": 0.7142012588871117, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8956521739130435, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26086956521739135, "hi": 0.4304347826086956, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6610423051232567, "hi": 0.8192780107206744, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.10330224245985113, "hi": 0.35710062944355586, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 84, "matches": 31, "known": 39, "missed": 8, "counts": { "yes_true": 31, "yes_false": 53, "no_true": 8, "no_false": 138 }, "rates": { "ppv": 0.36904761904761907, "npv": 0.9452054794520548, "sens": 0.7948717948717948, "spec": 0.7225130890052356, "youden": 0.5173848838770305, "accuracy": 0.7347826086956522, "balancedAccuracy": 0.7586924419385153 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 91, "matches": 32, "known": 39, "missed": 7, "counts": { "yes_true": 32, "yes_false": 59, "no_true": 7, "no_false": 132 }, "rates": { "ppv": 0.3516483516483517, "npv": 0.9496402877697842, "sens": 0.8205128205128205, "spec": 0.6910994764397905, "youden": 0.5116122969526109, "accuracy": 0.7130434782608696, "balancedAccuracy": 0.7558061484763055 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 920, "saved": 534 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 34, "known": 39, "missed": 5, "counts": { "yes_true": 34, "yes_false": 88, "no_true": 5, "no_false": 103 }, "rates": { "ppv": 0.2786885245901639, "npv": 0.9537037037037037, "sens": 0.8717948717948718, "spec": 0.5392670157068062, "youden": 0.41106188750167805, "accuracy": 0.5956521739130435, "balancedAccuracy": 0.705530943750839 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 46, "matches": 24, "known": 39, "missed": 15, "counts": { "yes_true": 24, "yes_false": 22, "no_true": 15, "no_false": 169 }, "rates": { "ppv": 0.5217391304347826, "npv": 0.9184782608695652, "sens": 0.6153846153846154, "spec": 0.8848167539267016, "youden": 0.500201369311317, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.7501006846556585 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 46, "matches": 24, "known": 25, "missed": 1, "counts": { "yes_true": 24, "yes_false": 22, "no_true": 1, "no_false": 5 }, "rates": { "ppv": 0.5217391304347826, "npv": 0.8333333333333334, "sens": 0.96, "spec": 0.18518518518518517, "youden": 0.14518518518518508, "accuracy": 0.5576923076923077, "balancedAccuracy": 0.5725925925925925 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 1, "removedAtSecond": 6, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 690, "saved": 304 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [ "R42841227|H5" ], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 46, "matches": 24, "known": 39, "missed": 15, "counts": { "yes_true": 24, "yes_false": 22, "no_true": 15, "no_false": 169 }, "rates": { "ppv": 0.5217391304347826, "npv": 0.9184782608695652, "sens": 0.6153846153846154, "spec": 0.8848167539267016, "youden": 0.500201369311317, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.7501006846556585 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3422519509476031, "npv": 0.0777702962677953, "sens": 0.07692307692307698, "spec": 0.38743455497382195, "youden": 0.46435763189689894, "accuracy": 0.33478260869565224, "balancedAccuracy": 0.23217881594844947 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.35555555555555557, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.19991795001962565, "hi": 0.4831823264951516, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8670512544582463, "hi": 0.9615437158469945, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.013156502054978137, "hi": 0.14288081188141116, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4594594594594595, "hi": 0.7777777777777778, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.16675531914893624, "hi": 0.3333333333333333, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8393782383419689, "hi": 0.925531914893617, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.31121589827640894, "hi": 0.46842927631578946, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3359978946454437, "hi": 0.6739150342411211, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.20327980513962227, "hi": 0.7385816151202748, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7913043478260869, "hi": 0.8869565217391304, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.25217391304347825, "hi": 0.42173913043478256, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6679989473227218, "hi": 0.8369575171205605, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.10163990256981113, "hi": 0.3692908075601374, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 34, "known": 39, "missed": 5, "counts": { "yes_true": 34, "yes_false": 90, "no_true": 5, "no_false": 101 }, "rates": { "ppv": 0.27419354838709675, "npv": 0.9528301886792453, "sens": 0.8717948717948718, "spec": 0.5287958115183246, "youden": 0.40059068331319647, "accuracy": 0.5869565217391305, "balancedAccuracy": 0.7002953416565982 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 128, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 93, "no_true": 4, "no_false": 98 }, "rates": { "ppv": 0.2734375, "npv": 0.9607843137254902, "sens": 0.8974358974358975, "spec": 0.5130890052356021, "youden": 0.41052490267149944, "accuracy": 0.5782608695652174, "balancedAccuracy": 0.7052624513357497 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 920, "saved": 534 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 87, "no_true": 4, "no_false": 104 }, "rates": { "ppv": 0.28688524590163933, "npv": 0.9629629629629629, "sens": 0.8974358974358975, "spec": 0.5445026178010471, "youden": 0.4419385152369446, "accuracy": 0.6043478260869565, "balancedAccuracy": 0.7209692576184723 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 48, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 23, "no_true": 14, "no_false": 168 }, "rates": { "ppv": 0.5208333333333334, "npv": 0.9230769230769231, "sens": 0.6410256410256411, "spec": 0.8795811518324608, "youden": 0.5206067928581017, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.7603033964290509 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 48, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 23, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.5208333333333334, "npv": 1, "sens": 1, "spec": 0.14814814814814814, "youden": 0.14814814814814814, "accuracy": 0.5576923076923077, "balancedAccuracy": 0.5740740740740741 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 690, "saved": 304 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 48, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 23, "no_true": 14, "no_false": 168 }, "rates": { "ppv": 0.5208333333333334, "npv": 0.9230769230769231, "sens": 0.6410256410256411, "spec": 0.8795811518324608, "youden": 0.5206067928581017, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.7603033964290509 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.34134615384615385, "npv": 0.08236895847515324, "sens": 0.10256410256410264, "spec": 0.38219895287958117, "youden": 0.4847630554436837, "accuracy": 0.33478260869565224, "balancedAccuracy": 0.24238152772184185 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.35555555555555557, "hi": 0.68, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.20634966135056804, "hi": 0.4800505871725382, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8715083798882681, "hi": 0.9664804469273743, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.017186240765654268, "hi": 0.14797719072380217, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8382218201416692, "hi": 0.918918918918919, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30768229166666655, "hi": 0.46154970760233915, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.352140489522451, "hi": 0.6936219614439968, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.2189930117905491, "hi": 0.7658707208891494, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7869565217391304, "hi": 0.8869565217391304, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.25217391304347825, "hi": 0.4217391304347826, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6760702447612255, "hi": 0.8468109807219983, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.10949650589527456, "hi": 0.3829353604445747, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 125, "matches": 36, "known": 39, "missed": 3, "counts": { "yes_true": 36, "yes_false": 89, "no_true": 3, "no_false": 102 }, "rates": { "ppv": 0.288, "npv": 0.9714285714285714, "sens": 0.9230769230769231, "spec": 0.5340314136125655, "youden": 0.4571083366894886, "accuracy": 0.6, "balancedAccuracy": 0.7285541683447443 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 48, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 23, "no_true": 14, "no_false": 168 }, "rates": { "ppv": 0.5208333333333334, "npv": 0.9230769230769231, "sens": 0.6410256410256411, "spec": 0.8795811518324608, "youden": 0.5206067928581017, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.7603033964290509 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 48, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 23, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.5208333333333334, "npv": 1, "sens": 1, "spec": 0.14814814814814814, "youden": 0.14814814814814814, "accuracy": 0.5576923076923077, "balancedAccuracy": 0.5740740740740741 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 230, "survivors": 52, "additionalModels": 4, "second": 208, "total": 438, "fullPanel": 920, "saved": 482 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 48, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 23, "no_true": 14, "no_false": 168 }, "rates": { "ppv": 0.5208333333333334, "npv": 0.9230769230769231, "sens": 0.6410256410256411, "spec": 0.8795811518324608, "youden": 0.5206067928581017, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.7603033964290509 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.34134615384615385, "npv": 0.08236895847515324, "sens": 0.10256410256410264, "spec": 0.38219895287958117, "youden": 0.4847630554436837, "accuracy": 0.33478260869565224, "balancedAccuracy": 0.24238152772184185 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.35555555555555557, "hi": 0.68, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.20634966135056804, "hi": 0.4800505871725382, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8715083798882681, "hi": 0.9664804469273743, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.017186240765654268, "hi": 0.14797719072380217, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8382218201416692, "hi": 0.918918918918919, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30768229166666655, "hi": 0.46154970760233915, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.352140489522451, "hi": 0.6936219614439968, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.2189930117905491, "hi": 0.7658707208891494, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7869565217391304, "hi": 0.8869565217391304, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.25217391304347825, "hi": 0.4217391304347826, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6760702447612255, "hi": 0.8468109807219983, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.10949650589527456, "hi": 0.3829353604445747, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 126, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 91, "no_true": 4, "no_false": 100 }, "rates": { "ppv": 0.2777777777777778, "npv": 0.9615384615384616, "sens": 0.8974358974358975, "spec": 0.5235602094240838, "youden": 0.42099610685998123, "accuracy": 0.5869565217391305, "balancedAccuracy": 0.7104980534299906 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 920, "saved": 534 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 129, "matches": 36, "known": 39, "missed": 3, "counts": { "yes_true": 36, "yes_false": 93, "no_true": 3, "no_false": 98 }, "rates": { "ppv": 0.27906976744186046, "npv": 0.9702970297029703, "sens": 0.9230769230769231, "spec": 0.5130890052356021, "youden": 0.4361659283125252, "accuracy": 0.5826086956521739, "balancedAccuracy": 0.7180829641562626 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 25, "known": 25, "missed": 0, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4807692307692308, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4807692307692308, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 12, "known": 25, "missed": 13, "counts": { "yes_true": 12, "yes_false": 15, "no_true": 13, "no_false": 12 }, "rates": { "ppv": 0.4444444444444444, "npv": 0.48, "sens": 0.48, "spec": 0.4444444444444444, "youden": -0.0755555555555556, "accuracy": 0.46153846153846156, "balancedAccuracy": 0.4622222222222222 } }, "lostAtFirst": 14, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 4, "second": 208, "total": 438, "fullPanel": 1150, "saved": 712 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H2", "R42647119|H4", "R42664348|H3", "R42689491|H2", "R42744804|H1", "R42775698|H1", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42669067|H4", "R42669067|H5", "R42811065|H2" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 25, "known": 39, "missed": 14, "counts": { "yes_true": 25, "yes_false": 27, "no_true": 14, "no_false": 164 }, "rates": { "ppv": 0.4807692307692308, "npv": 0.9213483146067416, "sens": 0.6410256410256411, "spec": 0.8586387434554974, "youden": 0.4996643844811386, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.7498321922405693 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.3012820512820513, "npv": 0.08064035000497172, "sens": 0.10256410256410264, "spec": 0.3612565445026178, "youden": 0.46382064706672055, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.23191032353336027 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3199787234042553, "hi": 0.6458333333333334, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1695712398981164, "hi": 0.43758369449155193, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8690427400468385, "hi": 0.9657191558441558, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.014948210922787251, "hi": 0.14656352799139716, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.4864864864864865, "hi": 0.8055555555555556, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.13890503875968999, "hi": 0.3589999999999999, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8071065989847716, "hi": 0.9027027027027027, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2841928920238741, "hi": 0.44385094947069736, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33139567308844886, "hi": 0.6769356684981684, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.19750546701011557, "hi": 0.7497454174264357, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7652173913043478, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.40869565217391307, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6656978365442244, "hi": 0.8384678342490842, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.09875273350505778, "hi": 0.3748727087132179, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 85, "matches": 32, "known": 39, "missed": 7, "counts": { "yes_true": 32, "yes_false": 53, "no_true": 7, "no_false": 138 }, "rates": { "ppv": 0.3764705882352941, "npv": 0.9517241379310345, "sens": 0.8205128205128205, "spec": 0.7225130890052356, "youden": 0.5430259095180561, "accuracy": 0.7391304347826086, "balancedAccuracy": 0.771512954759028 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 89, "no_true": 4, "no_false": 102 }, "rates": { "ppv": 0.28225806451612906, "npv": 0.9622641509433962, "sens": 0.8974358974358975, "spec": 0.5340314136125655, "youden": 0.43146731104846303, "accuracy": 0.5956521739130435, "balancedAccuracy": 0.7157336555242315 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 87, "no_true": 4, "no_false": 104 }, "rates": { "ppv": 0.28688524590163933, "npv": 0.9629629629629629, "sens": 0.8974358974358975, "spec": 0.5445026178010471, "youden": 0.4419385152369446, "accuracy": 0.6043478260869565, "balancedAccuracy": 0.7209692576184723 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 89, "no_true": 4, "no_false": 102 }, "rates": { "ppv": 0.28225806451612906, "npv": 0.9622641509433962, "sens": 0.8974358974358975, "spec": 0.5340314136125655, "youden": 0.43146731104846303, "accuracy": 0.5956521739130435, "balancedAccuracy": 0.7157336555242315 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 41, "matches": 20, "known": 39, "missed": 19, "counts": { "yes_true": 20, "yes_false": 21, "no_true": 19, "no_false": 170 }, "rates": { "ppv": 0.4878048780487805, "npv": 0.8994708994708994, "sens": 0.5128205128205128, "spec": 0.8900523560209425, "youden": 0.4028728688414551, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.7014364344207276 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 41, "matches": 20, "known": 21, "missed": 1, "counts": { "yes_true": 20, "yes_false": 21, "no_true": 1, "no_false": 3 }, "rates": { "ppv": 0.4878048780487805, "npv": 0.75, "sens": 0.9523809523809523, "spec": 0.125, "youden": 0.07738095238095233, "accuracy": 0.5111111111111111, "balancedAccuracy": 0.5386904761904762 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 1, "removedAtSecond": 4, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 690, "saved": 325 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [ "R42669067|H5" ], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 41, "matches": 20, "known": 39, "missed": 19, "counts": { "yes_true": 20, "yes_false": 21, "no_true": 19, "no_false": 170 }, "rates": { "ppv": 0.4878048780487805, "npv": 0.8994708994708994, "sens": 0.5128205128205128, "spec": 0.8900523560209425, "youden": 0.4028728688414551, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.7014364344207276 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.30831769856160096, "npv": 0.058762934869129535, "sens": -0.02564102564102566, "spec": 0.39267015706806285, "youden": 0.3670291314270371, "accuracy": 0.32173913043478264, "balancedAccuracy": 0.18351456571351854 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3333333333333333, "hi": 0.6511627906976745, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1722069071238159, "hi": 0.4529920523609163, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8445477463925567, "hi": 0.946524064171123, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.004081044134078304, "hi": 0.12263903012064774, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.38461538461538464, "hi": 0.6587398373983729, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2580943847072879, "hi": 0.23817733990147738, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8615384615384616, "hi": 0.9214761943717277, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.31520988469853467, "hi": 0.47594219505984203, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2613462262704635, "hi": 0.5667973553614563, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10783770546993514, "hi": 0.6554632348152594, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7782608695652173, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23478260869565226, "hi": 0.4088043478260864, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6306731131352318, "hi": 0.7833986776807281, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05391885273496757, "hi": 0.3277316174076297, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 128, "matches": 36, "known": 39, "missed": 3, "counts": { "yes_true": 36, "yes_false": 92, "no_true": 3, "no_false": 99 }, "rates": { "ppv": 0.28125, "npv": 0.9705882352941176, "sens": 0.9230769230769231, "spec": 0.518324607329843, "youden": 0.441401530406766, "accuracy": 0.5869565217391305, "balancedAccuracy": 0.720700765203383 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 920, "saved": 555 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 85, "matches": 30, "known": 39, "missed": 9, "counts": { "yes_true": 30, "yes_false": 55, "no_true": 9, "no_false": 136 }, "rates": { "ppv": 0.35294117647058826, "npv": 0.9379310344827586, "sens": 0.7692307692307693, "spec": 0.7120418848167539, "youden": 0.4812726540475232, "accuracy": 0.7217391304347827, "balancedAccuracy": 0.7406363270237616 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 79, "matches": 29, "known": 39, "missed": 10, "counts": { "yes_true": 29, "yes_false": 50, "no_true": 10, "no_false": 141 }, "rates": { "ppv": 0.3670886075949367, "npv": 0.9337748344370861, "sens": 0.7435897435897436, "spec": 0.7382198952879581, "youden": 0.48180963887770156, "accuracy": 0.7391304347826086, "balancedAccuracy": 0.7409048194388508 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 84, "matches": 31, "known": 39, "missed": 8, "counts": { "yes_true": 31, "yes_false": 53, "no_true": 8, "no_false": 138 }, "rates": { "ppv": 0.36904761904761907, "npv": 0.9452054794520548, "sens": 0.7948717948717948, "spec": 0.7225130890052356, "youden": 0.5173848838770305, "accuracy": 0.7347826086956522, "balancedAccuracy": 0.7586924419385153 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 38, "matches": 20, "known": 39, "missed": 19, "counts": { "yes_true": 20, "yes_false": 18, "no_true": 19, "no_false": 173 }, "rates": { "ppv": 0.5263157894736842, "npv": 0.9010416666666666, "sens": 0.5128205128205128, "spec": 0.9057591623036649, "youden": 0.4185796751241777, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.7092898375620889 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 38, "matches": 20, "known": 21, "missed": 1, "counts": { "yes_true": 20, "yes_false": 18, "no_true": 1, "no_false": 6 }, "rates": { "ppv": 0.5263157894736842, "npv": 0.8571428571428571, "sens": 0.9523809523809523, "spec": 0.25, "youden": 0.20238095238095233, "accuracy": 0.5777777777777777, "balancedAccuracy": 0.6011904761904762 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 1, "removedAtSecond": 7, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 690, "saved": 325 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42822442|H5", "R42826184|H1", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [ "R42669067|H5" ], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 38, "matches": 20, "known": 39, "missed": 19, "counts": { "yes_true": 20, "yes_false": 18, "no_true": 19, "no_false": 173 }, "rates": { "ppv": 0.5263157894736842, "npv": 0.9010416666666666, "sens": 0.5128205128205128, "spec": 0.9057591623036649, "youden": 0.4185796751241777, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.7092898375620889 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.34682860998650467, "npv": 0.060333702064896744, "sens": -0.02564102564102566, "spec": 0.4083769633507853, "youden": 0.38273593770975967, "accuracy": 0.33478260869565224, "balancedAccuracy": 0.19136796885487983 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.36357323232323235, "hi": 0.6923076923076923, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.19949943525039804, "hi": 0.4939035353026178, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8465608465608465, "hi": 0.9476439790575916, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.001700898308041221, "hi": 0.12364923247500514, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.38461538461538464, "hi": 0.6587398373983729, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2580943847072879, "hi": 0.23817733990147738, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8756218905472637, "hi": 0.9375, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3315789473684211, "hi": 0.4852941176470588, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.27172836525914723, "hi": 0.580564187810807, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12287868932043769, "hi": 0.6701396183473389, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7913043478260869, "hi": 0.8826086956521739, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.24782608695652175, "hi": 0.4260869565217391, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6358641826295737, "hi": 0.7902820939054035, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.061439344660218845, "hi": 0.33506980917366946, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 91, "matches": 32, "known": 39, "missed": 7, "counts": { "yes_true": 32, "yes_false": 59, "no_true": 7, "no_false": 132 }, "rates": { "ppv": 0.3516483516483517, "npv": 0.9496402877697842, "sens": 0.8205128205128205, "spec": 0.6910994764397905, "youden": 0.5116122969526109, "accuracy": 0.7130434782608696, "balancedAccuracy": 0.7558061484763055 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 920, "saved": 555 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 34, "known": 39, "missed": 5, "counts": { "yes_true": 34, "yes_false": 88, "no_true": 5, "no_false": 103 }, "rates": { "ppv": 0.2786885245901639, "npv": 0.9537037037037037, "sens": 0.8717948717948718, "spec": 0.5392670157068062, "youden": 0.41106188750167805, "accuracy": 0.5956521739130435, "balancedAccuracy": 0.705530943750839 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 34, "known": 39, "missed": 5, "counts": { "yes_true": 34, "yes_false": 90, "no_true": 5, "no_false": 101 }, "rates": { "ppv": 0.27419354838709675, "npv": 0.9528301886792453, "sens": 0.8717948717948718, "spec": 0.5287958115183246, "youden": 0.40059068331319647, "accuracy": 0.5869565217391305, "balancedAccuracy": 0.7002953416565982 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 41, "matches": 20, "known": 39, "missed": 19, "counts": { "yes_true": 20, "yes_false": 21, "no_true": 19, "no_false": 170 }, "rates": { "ppv": 0.4878048780487805, "npv": 0.8994708994708994, "sens": 0.5128205128205128, "spec": 0.8900523560209425, "youden": 0.4028728688414551, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.7014364344207276 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 41, "matches": 20, "known": 21, "missed": 1, "counts": { "yes_true": 20, "yes_false": 21, "no_true": 1, "no_false": 3 }, "rates": { "ppv": 0.4878048780487805, "npv": 0.75, "sens": 0.9523809523809523, "spec": 0.125, "youden": 0.07738095238095233, "accuracy": 0.5111111111111111, "balancedAccuracy": 0.5386904761904762 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 1, "removedAtSecond": 4, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 690, "saved": 325 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [ "R42669067|H5" ], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 41, "matches": 20, "known": 39, "missed": 19, "counts": { "yes_true": 20, "yes_false": 21, "no_true": 19, "no_false": 170 }, "rates": { "ppv": 0.4878048780487805, "npv": 0.8994708994708994, "sens": 0.5128205128205128, "spec": 0.8900523560209425, "youden": 0.4028728688414551, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.7014364344207276 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.30831769856160096, "npv": 0.058762934869129535, "sens": -0.02564102564102566, "spec": 0.39267015706806285, "youden": 0.3670291314270371, "accuracy": 0.32173913043478264, "balancedAccuracy": 0.18351456571351854 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3333333333333333, "hi": 0.6585365853658537, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.17113607188703464, "hi": 0.45396885348650107, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8448989419814847, "hi": 0.9468085106382979, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.003901661406671333, "hi": 0.12279126746142034, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.38461538461538464, "hi": 0.6587398373983729, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2580943847072879, "hi": 0.23817733990147738, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8622404996481351, "hi": 0.9234972677595629, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.31720430107526876, "hi": 0.47264885251163535, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2620563725490196, "hi": 0.5696839065293927, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.1027920498494738, "hi": 0.6632024985681557, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7782608695652173, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23478260869565226, "hi": 0.408695652173913, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6310281862745097, "hi": 0.7848419532646964, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.0513960249247369, "hi": 0.33160124928407786, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 128, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 93, "no_true": 4, "no_false": 98 }, "rates": { "ppv": 0.2734375, "npv": 0.9607843137254902, "sens": 0.8974358974358975, "spec": 0.5130890052356021, "youden": 0.41052490267149944, "accuracy": 0.5782608695652174, "balancedAccuracy": 0.7052624513357497 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 920, "saved": 555 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 87, "no_true": 4, "no_false": 104 }, "rates": { "ppv": 0.28688524590163933, "npv": 0.9629629629629629, "sens": 0.8974358974358975, "spec": 0.5445026178010471, "youden": 0.4419385152369446, "accuracy": 0.6043478260869565, "balancedAccuracy": 0.7209692576184723 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 42, "matches": 20, "known": 39, "missed": 19, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 19, "no_false": 169 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.898936170212766, "sens": 0.5128205128205128, "spec": 0.8848167539267016, "youden": 0.39763726674721434, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.6988186333736072 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 42, "matches": 20, "known": 21, "missed": 1, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 1, "no_false": 2 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.6666666666666666, "sens": 0.9523809523809523, "spec": 0.08333333333333333, "youden": 0.03571428571428559, "accuracy": 0.4888888888888889, "balancedAccuracy": 0.5178571428571428 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 1, "removedAtSecond": 3, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 690, "saved": 325 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [ "R42669067|H5" ], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 42, "matches": 20, "known": 39, "missed": 19, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 19, "no_false": 169 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.898936170212766, "sens": 0.5128205128205128, "spec": 0.8848167539267016, "youden": 0.39763726674721434, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.6988186333736072 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.29670329670329665, "npv": 0.058228205610996064, "sens": -0.02564102564102566, "spec": 0.38743455497382195, "youden": 0.3617935293327963, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.18089676466639815 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.31815410199556543, "hi": 0.6383660665575556, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16184133848607535, "hi": 0.43604408412764184, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8437373310810811, "hi": 0.946524064171123, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.004889192335755554, "hi": 0.1217698663709301, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.38461538461538464, "hi": 0.6587398373983729, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2580943847072879, "hi": 0.23817733990147738, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8571251768033946, "hi": 0.9157894736842105, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3115494564660035, "hi": 0.4674000701262272, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.254207552339614, "hi": 0.5607325427648503, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10132122849139655, "hi": 0.653258529147704, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7739130434782608, "hi": 0.8652173913043478, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.4043478260869566, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6271037761698071, "hi": 0.7803662713824251, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.050660614245698274, "hi": 0.326629264573852, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 125, "matches": 36, "known": 39, "missed": 3, "counts": { "yes_true": 36, "yes_false": 89, "no_true": 3, "no_false": 102 }, "rates": { "ppv": 0.288, "npv": 0.9714285714285714, "sens": 0.9230769230769231, "spec": 0.5340314136125655, "youden": 0.4571083366894886, "accuracy": 0.6, "balancedAccuracy": 0.7285541683447443 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 920, "saved": 555 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 126, "matches": 35, "known": 39, "missed": 4, "counts": { "yes_true": 35, "yes_false": 91, "no_true": 4, "no_false": 100 }, "rates": { "ppv": 0.2777777777777778, "npv": 0.9615384615384616, "sens": 0.8974358974358975, "spec": 0.5235602094240838, "youden": 0.42099610685998123, "accuracy": 0.5869565217391305, "balancedAccuracy": 0.7104980534299906 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 42, "matches": 20, "known": 39, "missed": 19, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 19, "no_false": 169 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.898936170212766, "sens": 0.5128205128205128, "spec": 0.8848167539267016, "youden": 0.39763726674721434, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.6988186333736072 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 42, "matches": 20, "known": 21, "missed": 1, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 1, "no_false": 2 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.6666666666666666, "sens": 0.9523809523809523, "spec": 0.08333333333333333, "youden": 0.03571428571428559, "accuracy": 0.4888888888888889, "balancedAccuracy": 0.5178571428571428 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 1, "removedAtSecond": 3, "workload": { "first": 230, "survivors": 45, "additionalModels": 4, "second": 180, "total": 410, "fullPanel": 920, "saved": 510 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [ "R42669067|H5" ], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 42, "matches": 20, "known": 39, "missed": 19, "counts": { "yes_true": 20, "yes_false": 22, "no_true": 19, "no_false": 169 }, "rates": { "ppv": 0.47619047619047616, "npv": 0.898936170212766, "sens": 0.5128205128205128, "spec": 0.8848167539267016, "youden": 0.39763726674721434, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.6988186333736072 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.29670329670329665, "npv": 0.058228205610996064, "sens": -0.02564102564102566, "spec": 0.38743455497382195, "youden": 0.3617935293327963, "accuracy": 0.31739130434782614, "balancedAccuracy": 0.18089676466639815 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.31815410199556543, "hi": 0.6383660665575556, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.16184133848607535, "hi": 0.43604408412764184, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.8437373310810811, "hi": 0.946524064171123, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.004889192335755554, "hi": 0.1217698663709301, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.38461538461538464, "hi": 0.6587398373983729, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2580943847072879, "hi": 0.23817733990147738, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8571251768033946, "hi": 0.9157894736842105, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.3115494564660035, "hi": 0.4674000701262272, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.254207552339614, "hi": 0.5607325427648503, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.10132122849139655, "hi": 0.653258529147704, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7739130434782608, "hi": 0.8652173913043478, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.4043478260869566, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6271037761698071, "hi": 0.7803662713824251, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.050660614245698274, "hi": 0.326629264573852, "validReplicates": 2000, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 129, "matches": 36, "known": 39, "missed": 3, "counts": { "yes_true": 36, "yes_false": 93, "no_true": 3, "no_false": 98 }, "rates": { "ppv": 0.27906976744186046, "npv": 0.9702970297029703, "sens": 0.9230769230769231, "spec": 0.5130890052356021, "youden": 0.4361659283125252, "accuracy": 0.5826086956521739, "balancedAccuracy": 0.7180829641562626 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 21, "known": 21, "missed": 0, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.4666666666666667, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 10, "known": 21, "missed": 11, "counts": { "yes_true": 10, "yes_false": 13, "no_true": 11, "no_false": 11 }, "rates": { "ppv": 0.43478260869565216, "npv": 0.5, "sens": 0.47619047619047616, "spec": 0.4583333333333333, "youden": -0.06547619047619047, "accuracy": 0.4666666666666667, "balancedAccuracy": 0.46726190476190477 } }, "lostAtFirst": 18, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 4, "second": 180, "total": 410, "fullPanel": 1150, "saved": 740 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42627902|H1", "R42630044|H2", "R42647119|H4", "R42647635|H4", "R42647635|H5", "R42689491|H2", "R42689491|H3", "R42744804|H1", "R42775698|H1", "R42841227|H5", "R42842666|H2", "R42842666|H3", "R42842666|H4", "R42844664|H5", "R42723144|H5", "R42669067|H4", "R42811065|H2", "R42806572|H5" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 24, "no_true": 18, "no_false": 167 }, "rates": { "ppv": 0.4666666666666667, "npv": 0.9027027027027027, "sens": 0.5384615384615384, "spec": 0.8743455497382199, "youden": 0.4128070881997583, "accuracy": 0.8173913043478261, "balancedAccuracy": 0.7064035440998792 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 21, "known": 39, "missed": 18, "counts": { "yes_true": 21, "yes_false": 96, "no_true": 18, "no_false": 95 }, "rates": { "ppv": 0.1794871794871795, "npv": 0.8407079646017699, "sens": 0.5384615384615384, "spec": 0.4973821989528796, "youden": 0.03584373741441804, "accuracy": 0.5043478260869565, "balancedAccuracy": 0.517921868707209 } }, "differences": { "ppv": 0.28717948717948716, "npv": 0.06199473810093281, "sens": 0, "spec": 0.37696335078534027, "youden": 0.37696335078534027, "accuracy": 0.31304347826086965, "balancedAccuracy": 0.18848167539267013 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.3170731707317073, "hi": 0.6170212765957447, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0.10184792843691148, "hi": 0.2705052091313007, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.1581871678283925, "hi": 0.4214091053816792, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.849149177110071, "hi": 0.9505494505494505, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.7708285550458716, "hi": 0.9029359596713965, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": 0.001247107720444693, "hi": 0.12458072955063867, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.40625, "hi": 0.6842481203007517, "validReplicates": 2000, "degenerate": false }, "right_sens": { "lo": 0.39124396135265704, "hi": 0.6875, "validReplicates": 2000, "degenerate": false }, "difference_sens": { "lo": -0.2142857142857143, "hi": 0.24999999999999994, "validReplicates": 2000, "degenerate": false }, "left_spec": { "lo": 0.8449197860962567, "hi": 0.9060816643646409, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.4292508710801394, "hi": 0.5634612269143535, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.30364086599688694, "hi": 0.4550561797752809, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.2589507428618221, "hi": 0.5678277332549073, "validReplicates": 2000, "degenerate": false }, "right_youden": { "lo": -0.13685302424473225, "hi": 0.20161769497911358, "validReplicates": 2000, "degenerate": false }, "difference_youden": { "lo": 0.12576650943396223, "hi": 0.6559984895120835, "validReplicates": 2000, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.4391304347826087, "hi": 0.5695652173913044, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.22608695652173916, "hi": 0.4, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.629475371430911, "hi": 0.7839138666274537, "validReplicates": 2000, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.4315734878776339, "hi": 0.6008088474895568, "validReplicates": 2000, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.06288325471698111, "hi": 0.32799924475604175, "validReplicates": 2000, "degenerate": false } } } } } ] }, { "handling": "strict", "endpoint": "strict", "totalOriginal": 250, "rows": [ { "task": "R42581103", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H2", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42581103", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42581103", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42601407", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42601407", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42601407", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42605193", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42605193", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42608398", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42608398", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42627902", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42627902", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42630044", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42630044", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42639736", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42639736", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42639736", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642518", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642518", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642518", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642521", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42642521", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42642521", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42647119", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647119", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H1", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42647635", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42647635", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42663085", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42663085", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42664348", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42664348", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42665751", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42665751", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42689491", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42689491", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42689491", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42696595", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42696595", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42698274", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42698274", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42717261", "id": "H5", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42723546", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42723546", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723546", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42736449", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42744804", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42744804", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42759510", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775695", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775695", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42775698", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42775698", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42775698", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42786310", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42786310", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42805954", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42805954", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H3", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42810392", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42810392", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42811497", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811497", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42821663", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42821663", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42821663", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42822442", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42822442", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42826184", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42826184", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42839094", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42839094", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42841227", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841227", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42842666", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42842666", "id": "H4", "y": 1, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42842666", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42844664", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42844664", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42844664", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42853938", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42853938", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42853938", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42858183", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42858183", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806945", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42806945", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42723144", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42808804", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42808804", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42808804", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42585340", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42585340", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42669067", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42669067", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42811065", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42811065", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H1", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H2", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": true } }, { "task": "R42841610", "id": "H3", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": false, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42841610", "id": "H5", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H1", "y": 1, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42806003", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H4", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806003", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H1", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H2", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": true, "gpt-6-astra": true, "gemini-4-argon-high": true, "grok-4.5": true, "claude-opus-5-5": true } }, { "task": "R42806572", "id": "H3", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H4", "y": 0, "yes": false, "coin": false, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": false, "gemini-4-argon-high": false, "grok-4.5": false, "claude-opus-5-5": false } }, { "task": "R42806572", "id": "H5", "y": 0, "yes": false, "coin": true, "forecasts": { "claude-sonnet-5-5": false, "gpt-6-astra": true, "gemini-4-argon-high": false, "grok-4.5": true, "claude-opus-5-5": false } } ], "results": [ { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 85, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 79, "no_true": 1, "no_false": 144 }, "rates": { "ppv": 0.07058823529411765, "npv": 0.993103448275862, "sens": 0.8571428571428571, "spec": 0.6457399103139013, "youden": 0.5028827674567584, "accuracy": 0.6521739130434783, "balancedAccuracy": 0.7514413837283792 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 117, "no_true": 0, "no_false": 106 }, "rates": { "ppv": 0.056451612903225805, "npv": 1, "sens": 1, "spec": 0.47533632286995514, "youden": 0.4753363228699552, "accuracy": 0.49130434782608695, "balancedAccuracy": 0.7376681614349776 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 115, "no_true": 0, "no_false": 108 }, "rates": { "ppv": 0.05737704918032787, "npv": 1, "sens": 1, "spec": 0.484304932735426, "youden": 0.4843049327354261, "accuracy": 0.5, "balancedAccuracy": 0.742152466367713 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 183 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9945652173913043, "sens": 0.8571428571428571, "spec": 0.820627802690583, "youden": 0.6777706598334401, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.83888532991672 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 6 }, "rates": { "ppv": 0.13043478260869565, "npv": 1, "sens": 1, "spec": 0.13043478260869565, "youden": 0.13043478260869557, "accuracy": 0.23076923076923078, "balancedAccuracy": 0.5652173913043478 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 6, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 690, "saved": 304 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 183 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9945652173913043, "sens": 0.8571428571428571, "spec": 0.820627802690583, "youden": 0.6777706598334401, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.83888532991672 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.10479375696767002, "npv": 0.029963447479799954, "sens": 0.42857142857142855, "spec": 0.33183856502242154, "youden": 0.7604099935938501, "accuracy": 0.3347826086956522, "balancedAccuracy": 0.38020499679692504 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.044444444444444446, "hi": 0.23081395348837186, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.022720708367370755, "hi": 0.20100594523630905, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9831460674157303, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.003101355411041262, "hi": 0.07240437158469948, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7792792792792793, "hi": 0.8590323388065679, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.25909090909090915, "hi": 0.40889393939393937, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.32014876718824087, "hi": 0.8509127548601233, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.0682544546759147, "hi": 1.3467679882525698, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.782608695652174, "hi": 0.8608695652173913, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.25652173913043474, "hi": 0.4173913043478261, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6600743835941204, "hi": 0.9254563774300616, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.03412722733795735, "hi": 0.6733839941262849, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 117, "no_true": 0, "no_false": 106 }, "rates": { "ppv": 0.056451612903225805, "npv": 1, "sens": 1, "spec": 0.47533632286995514, "youden": 0.4753363228699552, "accuracy": 0.49130434782608695, "balancedAccuracy": 0.7376681614349776 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 128, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 121, "no_true": 0, "no_false": 102 }, "rates": { "ppv": 0.0546875, "npv": 1, "sens": 1, "spec": 0.45739910313901344, "youden": 0.45739910313901344, "accuracy": 0.47391304347826085, "balancedAccuracy": 0.7286995515695067 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 920, "saved": 534 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 85, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 79, "no_true": 1, "no_false": 144 }, "rates": { "ppv": 0.07058823529411765, "npv": 0.993103448275862, "sens": 0.8571428571428571, "spec": 0.6457399103139013, "youden": 0.5028827674567584, "accuracy": 0.6521739130434783, "balancedAccuracy": 0.7514413837283792 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 79, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 73, "no_true": 1, "no_false": 150 }, "rates": { "ppv": 0.0759493670886076, "npv": 0.9933774834437086, "sens": 0.8571428571428571, "spec": 0.672645739910314, "youden": 0.529788597053171, "accuracy": 0.6782608695652174, "balancedAccuracy": 0.7648942985265855 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 40, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 34, "no_true": 1, "no_false": 189 }, "rates": { "ppv": 0.15, "npv": 0.9947368421052631, "sens": 0.8571428571428571, "spec": 0.8475336322869955, "youden": 0.7046764894298527, "accuracy": 0.8478260869565217, "balancedAccuracy": 0.8523382447149264 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 40, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 34, "no_true": 0, "no_false": 12 }, "rates": { "ppv": 0.15, "npv": 1, "sens": 1, "spec": 0.2608695652173913, "youden": 0.26086956521739135, "accuracy": 0.34615384615384615, "balancedAccuracy": 0.6304347826086957 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 12, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 690, "saved": 304 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42664348|H2", "R42665751|H3", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 40, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 34, "no_true": 1, "no_false": 189 }, "rates": { "ppv": 0.15, "npv": 0.9947368421052631, "sens": 0.8571428571428571, "spec": 0.8475336322869955, "youden": 0.7046764894298527, "accuracy": 0.8478260869565217, "balancedAccuracy": 0.8523382447149264 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.12435897435897436, "npv": 0.030135072193758772, "sens": 0.42857142857142855, "spec": 0.35874439461883406, "youden": 0.7873158231902627, "accuracy": 0.3608695652173913, "balancedAccuracy": 0.39365791159513136 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.049969512195121954, "hi": 0.2702702702702703, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.02820129360510486, "hi": 0.23363760556383492, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9836956521739131, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0025394385604245117, "hi": 0.07269503546099298, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8090858585858586, "hi": 0.8828828828828829, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.29146555238483496, "hi": 0.42986425339366513, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.34375, "hi": 0.875, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.08870844899992894, "hi": 1.3649886877828055, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.808695652173913, "hi": 0.8826086956521739, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.28695652173913044, "hi": 0.4348913043478255, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.671875, "hi": 0.9375, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.04435422449996447, "hi": 0.6824943438914027, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 84, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 78, "no_true": 1, "no_false": 145 }, "rates": { "ppv": 0.07142857142857142, "npv": 0.9931506849315068, "sens": 0.8571428571428571, "spec": 0.6502242152466368, "youden": 0.5073670723894939, "accuracy": 0.6565217391304348, "balancedAccuracy": 0.7536835361947469 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 91, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 85, "no_true": 1, "no_false": 138 }, "rates": { "ppv": 0.06593406593406594, "npv": 0.9928057553956835, "sens": 0.8571428571428571, "spec": 0.6188340807174888, "youden": 0.475976937860346, "accuracy": 0.6260869565217392, "balancedAccuracy": 0.737988468930173 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 920, "saved": 534 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 115, "no_true": 0, "no_false": 108 }, "rates": { "ppv": 0.05737704918032787, "npv": 1, "sens": 1, "spec": 0.484304932735426, "youden": 0.4843049327354261, "accuracy": 0.5, "balancedAccuracy": 0.742152466367713 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 183 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9945652173913043, "sens": 0.8571428571428571, "spec": 0.820627802690583, "youden": 0.6777706598334401, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.83888532991672 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 46, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 0, "no_false": 6 }, "rates": { "ppv": 0.13043478260869565, "npv": 1, "sens": 1, "spec": 0.13043478260869565, "youden": 0.13043478260869557, "accuracy": 0.23076923076923078, "balancedAccuracy": 0.5652173913043478 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 6, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 690, "saved": 304 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 46, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 40, "no_true": 1, "no_false": 183 }, "rates": { "ppv": 0.13043478260869565, "npv": 0.9945652173913043, "sens": 0.8571428571428571, "spec": 0.820627802690583, "youden": 0.6777706598334401, "accuracy": 0.8217391304347826, "balancedAccuracy": 0.83888532991672 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.10479375696767002, "npv": 0.029963447479799954, "sens": 0.42857142857142855, "spec": 0.33183856502242154, "youden": 0.7604099935938501, "accuracy": 0.3347826086956522, "balancedAccuracy": 0.38020499679692504 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.043478260869565216, "hi": 0.23259524987629865, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.023272565943373395, "hi": 0.20310278139939128, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9831460674157303, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.003176176878562616, "hi": 0.07246523697610652, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7782679738562092, "hi": 0.8616071428571429, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.25791503267973853, "hi": 0.4117844895805855, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.32130791505791506, "hi": 0.852017937219731, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.06415929203539837, "hi": 1.3484315476190476, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7782608695652173, "hi": 0.8609782608695646, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.25652173913043474, "hi": 0.4217391304347826, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6606539575289575, "hi": 0.9260089686098655, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.032079646017699186, "hi": 0.6742157738095238, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 117, "no_true": 0, "no_false": 106 }, "rates": { "ppv": 0.056451612903225805, "npv": 1, "sens": 1, "spec": 0.47533632286995514, "youden": 0.4753363228699552, "accuracy": 0.49130434782608695, "balancedAccuracy": 0.7376681614349776 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 2, "second": 104, "total": 334, "fullPanel": 690, "saved": 356 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 128, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 121, "no_true": 0, "no_false": 102 }, "rates": { "ppv": 0.0546875, "npv": 1, "sens": 1, "spec": 0.45739910313901344, "youden": 0.45739910313901344, "accuracy": 0.47391304347826085, "balancedAccuracy": 0.7286995515695067 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 920, "saved": 534 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 115, "no_true": 0, "no_false": 108 }, "rates": { "ppv": 0.05737704918032787, "npv": 1, "sens": 1, "spec": 0.484304932735426, "youden": 0.4843049327354261, "accuracy": 0.5, "balancedAccuracy": 0.742152466367713 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 48, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 42, "no_true": 1, "no_false": 181 }, "rates": { "ppv": 0.125, "npv": 0.9945054945054945, "sens": 0.8571428571428571, "spec": 0.8116591928251121, "youden": 0.6688020499679692, "accuracy": 0.8130434782608695, "balancedAccuracy": 0.8344010249839846 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 48, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 42, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.125, "npv": 1, "sens": 1, "spec": 0.08695652173913043, "youden": 0.08695652173913038, "accuracy": 0.19230769230769232, "balancedAccuracy": 0.5434782608695652 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 690, "saved": 304 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 48, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 42, "no_true": 1, "no_false": 181 }, "rates": { "ppv": 0.125, "npv": 0.9945054945054945, "sens": 0.8571428571428571, "spec": 0.8116591928251121, "youden": 0.6688020499679692, "accuracy": 0.8130434782608695, "balancedAccuracy": 0.8344010249839846 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.09935897435897437, "npv": 0.029903724593990155, "sens": 0.42857142857142855, "spec": 0.32286995515695066, "youden": 0.7514413837283792, "accuracy": 0.3260869565217391, "balancedAccuracy": 0.3757206918641896 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.041666666666666664, "hi": 0.2222222222222222, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021487029384756655, "hi": 0.1927558424669819, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9830508474576272, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.003306489629164094, "hi": 0.07234432234432242, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7699115044247787, "hi": 0.8506787330316742, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2511210762331839, "hi": 0.40178571428571425, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31415929203539816, "hi": 0.8414261367034107, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.055309734513274256, "hi": 1.336399784740493, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.24782608695652175, "hi": 0.41304347826086957, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6570796460176991, "hi": 0.9207130683517054, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.027654867256637128, "hi": 0.6681998923702465, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 125, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 118, "no_true": 0, "no_false": 105 }, "rates": { "ppv": 0.056, "npv": 1, "sens": 1, "spec": 0.47085201793721976, "youden": 0.47085201793721976, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.7354260089686099 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 48, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 42, "no_true": 1, "no_false": 181 }, "rates": { "ppv": 0.125, "npv": 0.9945054945054945, "sens": 0.8571428571428571, "spec": 0.8116591928251121, "youden": 0.6688020499679692, "accuracy": 0.8130434782608695, "balancedAccuracy": 0.8344010249839846 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 48, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 42, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.125, "npv": 1, "sens": 1, "spec": 0.08695652173913043, "youden": 0.08695652173913038, "accuracy": 0.19230769230769232, "balancedAccuracy": 0.5434782608695652 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 230, "survivors": 52, "additionalModels": 4, "second": 208, "total": 438, "fullPanel": 920, "saved": 482 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 48, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 42, "no_true": 1, "no_false": 181 }, "rates": { "ppv": 0.125, "npv": 0.9945054945054945, "sens": 0.8571428571428571, "spec": 0.8116591928251121, "youden": 0.6688020499679692, "accuracy": 0.8130434782608695, "balancedAccuracy": 0.8344010249839846 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.09935897435897437, "npv": 0.029903724593990155, "sens": 0.42857142857142855, "spec": 0.32286995515695066, "youden": 0.7514413837283792, "accuracy": 0.3260869565217391, "balancedAccuracy": 0.3757206918641896 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.041666666666666664, "hi": 0.2222222222222222, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.021487029384756655, "hi": 0.1927558424669819, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9830508474576272, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.003306489629164094, "hi": 0.07234432234432242, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7699115044247787, "hi": 0.8506787330316742, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2511210762331839, "hi": 0.40178571428571425, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31415929203539816, "hi": 0.8414261367034107, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.055309734513274256, "hi": 1.336399784740493, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7695652173913043, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.24782608695652175, "hi": 0.41304347826086957, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6570796460176991, "hi": 0.9207130683517054, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.027654867256637128, "hi": 0.6681998923702465, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 126, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 119, "no_true": 0, "no_false": 104 }, "rates": { "ppv": 0.05555555555555555, "npv": 1, "sens": 1, "spec": 0.4663677130044843, "youden": 0.4663677130044843, "accuracy": 0.4826086956521739, "balancedAccuracy": 0.7331838565022422 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 3, "second": 156, "total": 386, "fullPanel": 920, "saved": 534 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "gpt-6-astra", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 129, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 122, "no_true": 0, "no_false": 101 }, "rates": { "ppv": 0.05426356589147287, "npv": 1, "sens": 1, "spec": 0.452914798206278, "youden": 0.452914798206278, "accuracy": 0.46956521739130436, "balancedAccuracy": 0.726457399103139 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "conditional": { "hypotheses": 52, "questions": 41, "selected": 52, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.11538461538461539, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.11538461538461539, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 52, "questions": 41, "selected": 27, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 25, "no_true": 4, "no_false": 21 }, "rates": { "ppv": 0.07407407407407407, "npv": 0.84, "sens": 0.3333333333333333, "spec": 0.45652173913043476, "youden": -0.21014492753623193, "accuracy": 0.4423076923076923, "balancedAccuracy": 0.39492753623188404 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 52, "additionalModels": 4, "second": 208, "total": 438, "fullPanel": 1150, "saved": 712 }, "selectedIds": [ "R42581103|H2", "R42601407|H3", "R42605193|H4", "R42608398|H1", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42639736|H2", "R42639736|H3", "R42639736|H4", "R42642518|H4", "R42647119|H1", "R42647635|H1", "R42647635|H4", "R42647635|H5", "R42663085|H4", "R42664348|H2", "R42665751|H3", "R42665751|H5", "R42689491|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42775695|H5", "R42775698|H3", "R42786310|H3", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H3", "R42822442|H5", "R42826184|H3", "R42839094|H2", "R42839094|H5", "R42841227|H5", "R42842666|H1", "R42844664|H3", "R42844664|H5", "R42858183|H1", "R42806945|H4", "R42723144|H5", "R42808804|H4", "R42585340|H5", "R42669067|H1", "R42811065|H1", "R42841610|H2", "R42806003|H1", "R42806572|H2", "R42806572|H5" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 52, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 46, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.11538461538461539, "npv": 0.9943820224719101, "sens": 0.8571428571428571, "spec": 0.7937219730941704, "youden": 0.6508648302370275, "accuracy": 0.7956521739130434, "balancedAccuracy": 0.8254324151185137 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.08974358974358976, "npv": 0.02978025256040573, "sens": 0.42857142857142855, "spec": 0.3049327354260089, "youden": 0.7335041639974375, "accuracy": 0.308695652173913, "balancedAccuracy": 0.3667520819987187 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.038461538461538464, "hi": 0.20754716981132076, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.018311728242129476, "hi": 0.17795366350625416, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9825555895552835, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0041858674382342165, "hi": 0.07203705457185243, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7488516409381325, "hi": 0.8348214285714286, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.22972972972972971, "hi": 0.38392857142857145, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.290125704887218, "hi": 0.8243421052631579, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.03761061946902644, "hi": 1.3215859030837005, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7521739130434782, "hi": 0.8348913043478254, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.23043478260869565, "hi": 0.39565217391304347, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6450628524436091, "hi": 0.9121710526315789, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.01880530973451322, "hi": 0.6607929515418502, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 85, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 79, "no_true": 1, "no_false": 144 }, "rates": { "ppv": 0.07058823529411765, "npv": 0.993103448275862, "sens": 0.8571428571428571, "spec": 0.6457399103139013, "youden": 0.5028827674567584, "accuracy": 0.6521739130434783, "balancedAccuracy": 0.7514413837283792 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 117, "no_true": 0, "no_false": 106 }, "rates": { "ppv": 0.056451612903225805, "npv": 1, "sens": 1, "spec": 0.47533632286995514, "youden": 0.4753363228699552, "accuracy": 0.49130434782608695, "balancedAccuracy": 0.7376681614349776 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 115, "no_true": 0, "no_false": 108 }, "rates": { "ppv": 0.05737704918032787, "npv": 1, "sens": 1, "spec": 0.484304932735426, "youden": 0.4843049327354261, "accuracy": 0.5, "balancedAccuracy": 0.742152466367713 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 117, "no_true": 0, "no_false": 106 }, "rates": { "ppv": 0.056451612903225805, "npv": 1, "sens": 1, "spec": 0.47533632286995514, "youden": 0.4753363228699552, "accuracy": 0.49130434782608695, "balancedAccuracy": 0.7376681614349776 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 41, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 35, "no_true": 1, "no_false": 188 }, "rates": { "ppv": 0.14634146341463414, "npv": 0.9947089947089947, "sens": 0.8571428571428571, "spec": 0.8430493273542601, "youden": 0.7001921844971171, "accuracy": 0.8434782608695652, "balancedAccuracy": 0.8500960922485585 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 41, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 35, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.14634146341463414, "npv": 1, "sens": 1, "spec": 0.10256410256410256, "youden": 0.10256410256410264, "accuracy": 0.2222222222222222, "balancedAccuracy": 0.5512820512820513 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 690, "saved": 325 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 41, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 35, "no_true": 1, "no_false": 188 }, "rates": { "ppv": 0.14634146341463414, "npv": 0.9947089947089947, "sens": 0.8571428571428571, "spec": 0.8430493273542601, "youden": 0.7001921844971171, "accuracy": 0.8434782608695652, "balancedAccuracy": 0.8500960922485585 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1207004377736085, "npv": 0.030107224797490284, "sens": 0.42857142857142855, "spec": 0.3542600896860986, "youden": 0.7828315182575271, "accuracy": 0.3565217391304348, "balancedAccuracy": 0.39141575912876353 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.04878048780487805, "hi": 0.25, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.030284516429569645, "hi": 0.21814009661835743, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9837837837837838, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0024221695781146547, "hi": 0.07258350491878292, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8198198198198198, "hi": 0.8681818181818182, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2812362938596491, "hi": 0.43111111111111106, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.33628318584070804, "hi": 0.8648648648648649, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.09070796460176987, "hi": 1.3590219711359588, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8173913043478261, "hi": 0.8695652173913043, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.27826086956521745, "hi": 0.4347826086956522, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.668141592920354, "hi": 0.9324324324324325, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.045353982300884936, "hi": 0.6795109855679794, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 128, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 121, "no_true": 0, "no_false": 102 }, "rates": { "ppv": 0.0546875, "npv": 1, "sens": 1, "spec": 0.45739910313901344, "youden": 0.45739910313901344, "accuracy": 0.47391304347826085, "balancedAccuracy": 0.7286995515695067 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 920, "saved": 555 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 85, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 79, "no_true": 1, "no_false": 144 }, "rates": { "ppv": 0.07058823529411765, "npv": 0.993103448275862, "sens": 0.8571428571428571, "spec": 0.6457399103139013, "youden": 0.5028827674567584, "accuracy": 0.6521739130434783, "balancedAccuracy": 0.7514413837283792 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 79, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 73, "no_true": 1, "no_false": 150 }, "rates": { "ppv": 0.0759493670886076, "npv": 0.9933774834437086, "sens": 0.8571428571428571, "spec": 0.672645739910314, "youden": 0.529788597053171, "accuracy": 0.6782608695652174, "balancedAccuracy": 0.7648942985265855 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 84, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 78, "no_true": 1, "no_false": 145 }, "rates": { "ppv": 0.07142857142857142, "npv": 0.9931506849315068, "sens": 0.8571428571428571, "spec": 0.6502242152466368, "youden": 0.5073670723894939, "accuracy": 0.6565217391304348, "balancedAccuracy": 0.7536835361947469 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 38, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 32, "no_true": 1, "no_false": 191 }, "rates": { "ppv": 0.15789473684210525, "npv": 0.9947916666666666, "sens": 0.8571428571428571, "spec": 0.8565022421524664, "youden": 0.7136450992953236, "accuracy": 0.8565217391304348, "balancedAccuracy": 0.8568225496476618 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 38, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 32, "no_true": 0, "no_false": 7 }, "rates": { "ppv": 0.15789473684210525, "npv": 1, "sens": 1, "spec": 0.1794871794871795, "youden": 0.17948717948717952, "accuracy": 0.28888888888888886, "balancedAccuracy": 0.5897435897435898 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 7, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 690, "saved": 325 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42822442|H5", "R42826184|H1", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 38, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 32, "no_true": 1, "no_false": 191 }, "rates": { "ppv": 0.15789473684210525, "npv": 0.9947916666666666, "sens": 0.8571428571428571, "spec": 0.8565022421524664, "youden": 0.7136450992953236, "accuracy": 0.8565217391304348, "balancedAccuracy": 0.8568225496476618 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.13225371120107962, "npv": 0.03018989675516226, "sens": 0.42857142857142855, "spec": 0.36771300448430494, "youden": 0.7962844330557336, "accuracy": 0.3695652173913044, "balancedAccuracy": 0.3981422165278668 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.05405405405405406, "hi": 0.27088068181818153, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.0347209800848239, "hi": 0.23775407220132033, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.983957219251337, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0023106137492654886, "hi": 0.07269584139264998, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8303571428571429, "hi": 0.8844444444444445, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2946428571428572, "hi": 0.44196428571428575, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.35077821766806394, "hi": 0.8799999999999999, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.10215690690690692, "hi": 1.3761061946902655, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8260869565217391, "hi": 0.8869565217391304, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.29130434782608694, "hi": 0.4521739130434782, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.675389108834032, "hi": 0.94, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.05107845345345346, "hi": 0.6880530973451328, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 91, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 85, "no_true": 1, "no_false": 138 }, "rates": { "ppv": 0.06593406593406594, "npv": 0.9928057553956835, "sens": 0.8571428571428571, "spec": 0.6188340807174888, "youden": 0.475976937860346, "accuracy": 0.6260869565217392, "balancedAccuracy": 0.737988468930173 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 920, "saved": 555 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 115, "no_true": 0, "no_false": 108 }, "rates": { "ppv": 0.05737704918032787, "npv": 1, "sens": 1, "spec": 0.484304932735426, "youden": 0.4843049327354261, "accuracy": 0.5, "balancedAccuracy": 0.742152466367713 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 2, "second": 90, "total": 320, "fullPanel": 690, "saved": 370 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 124, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 117, "no_true": 0, "no_false": 106 }, "rates": { "ppv": 0.056451612903225805, "npv": 1, "sens": 1, "spec": 0.47533632286995514, "youden": 0.4753363228699552, "accuracy": 0.49130434782608695, "balancedAccuracy": 0.7376681614349776 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 41, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 35, "no_true": 1, "no_false": 188 }, "rates": { "ppv": 0.14634146341463414, "npv": 0.9947089947089947, "sens": 0.8571428571428571, "spec": 0.8430493273542601, "youden": 0.7001921844971171, "accuracy": 0.8434782608695652, "balancedAccuracy": 0.8500960922485585 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 41, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 35, "no_true": 0, "no_false": 4 }, "rates": { "ppv": 0.14634146341463414, "npv": 1, "sens": 1, "spec": 0.10256410256410256, "youden": 0.10256410256410264, "accuracy": 0.2222222222222222, "balancedAccuracy": 0.5512820512820513 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 4, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 690, "saved": 325 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 41, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 35, "no_true": 1, "no_false": 188 }, "rates": { "ppv": 0.14634146341463414, "npv": 0.9947089947089947, "sens": 0.8571428571428571, "spec": 0.8430493273542601, "youden": 0.7001921844971171, "accuracy": 0.8434782608695652, "balancedAccuracy": 0.8500960922485585 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1207004377736085, "npv": 0.030107224797490284, "sens": 0.42857142857142855, "spec": 0.3542600896860986, "youden": 0.7828315182575271, "accuracy": 0.3565217391304348, "balancedAccuracy": 0.39141575912876353 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.05, "hi": 0.2501329787234035, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.02923512328117591, "hi": 0.21838100686498843, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9836956521739131, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0024717109969613488, "hi": 0.07252397558849175, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8198198198198198, "hi": 0.8693840140589019, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.28378378378378377, "hi": 0.4305087194818135, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.3348214285714286, "hi": 0.8660714285714286, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.08888888888888878, "hi": 1.3588655920494486, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8173913043478261, "hi": 0.8739130434782608, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2824999999999999, "hi": 0.4347826086956522, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6674107142857143, "hi": 0.9330357142857143, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.04444444444444439, "hi": 0.6794327960247243, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 128, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 121, "no_true": 0, "no_false": 102 }, "rates": { "ppv": 0.0546875, "npv": 1, "sens": 1, "spec": 0.45739910313901344, "youden": 0.45739910313901344, "accuracy": 0.47391304347826085, "balancedAccuracy": 0.7286995515695067 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 920, "saved": 555 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 122, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 115, "no_true": 0, "no_false": 108 }, "rates": { "ppv": 0.05737704918032787, "npv": 1, "sens": 1, "spec": 0.484304932735426, "youden": 0.4843049327354261, "accuracy": 0.5, "balancedAccuracy": 0.742152466367713 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 42, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 1, "no_false": 187 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.9946808510638298, "sens": 0.8571428571428571, "spec": 0.8385650224215246, "youden": 0.6957078795643818, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.8478539397821909 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 42, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 0, "no_false": 3 }, "rates": { "ppv": 0.14285714285714285, "npv": 1, "sens": 1, "spec": 0.07692307692307693, "youden": 0.07692307692307687, "accuracy": 0.2, "balancedAccuracy": 0.5384615384615384 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 3, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 690, "saved": 325 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 42, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 1, "no_false": 187 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.9946808510638298, "sens": 0.8571428571428571, "spec": 0.8385650224215246, "youden": 0.6957078795643818, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.8478539397821909 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.11721611721611722, "npv": 0.030079081152325382, "sens": 0.42857142857142855, "spec": 0.3497757847533632, "youden": 0.7783472133247918, "accuracy": 0.3521739130434783, "balancedAccuracy": 0.3891736066623959 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.04878048780487805, "hi": 0.2444557823129251, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028228758283315405, "hi": 0.21272547212118192, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9836956521739131, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0024919393061872295, "hi": 0.07252397558849175, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8157894736842105, "hi": 0.8636363636363636, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27801455917504586, "hi": 0.4241379484486101, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.331858407079646, "hi": 0.8597758958053077, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.08614528023598807, "hi": 1.3542600896860986, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8130434782608695, "hi": 0.8652173913043478, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2781521739130434, "hi": 0.43043478260869567, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.665929203539823, "hi": 0.9298879479026538, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.043072640117994034, "hi": 0.6771300448430493, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 125, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 118, "no_true": 0, "no_false": 105 }, "rates": { "ppv": 0.056, "npv": 1, "sens": 1, "spec": 0.47085201793721976, "youden": 0.47085201793721976, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.7354260089686099 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 3, "second": 135, "total": 365, "fullPanel": 920, "saved": 555 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": false, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 126, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 119, "no_true": 0, "no_false": 104 }, "rates": { "ppv": 0.05555555555555555, "npv": 1, "sens": 1, "spec": 0.4663677130044843, "youden": 0.4663677130044843, "accuracy": 0.4826086956521739, "balancedAccuracy": 0.7331838565022422 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 42, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 1, "no_false": 187 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.9946808510638298, "sens": 0.8571428571428571, "spec": 0.8385650224215246, "youden": 0.6957078795643818, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.8478539397821909 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 42, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 0, "no_false": 3 }, "rates": { "ppv": 0.14285714285714285, "npv": 1, "sens": 1, "spec": 0.07692307692307693, "youden": 0.07692307692307687, "accuracy": 0.2, "balancedAccuracy": 0.5384615384615384 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 3, "workload": { "first": 230, "survivors": 45, "additionalModels": 4, "second": 180, "total": 410, "fullPanel": 920, "saved": 510 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 42, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 36, "no_true": 1, "no_false": 187 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.9946808510638298, "sens": 0.8571428571428571, "spec": 0.8385650224215246, "youden": 0.6957078795643818, "accuracy": 0.8391304347826087, "balancedAccuracy": 0.8478539397821909 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.11721611721611722, "npv": 0.030079081152325382, "sens": 0.42857142857142855, "spec": 0.3497757847533632, "youden": 0.7783472133247918, "accuracy": 0.3521739130434783, "balancedAccuracy": 0.3891736066623959 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.04878048780487805, "hi": 0.2444557823129251, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.028228758283315405, "hi": 0.21272547212118192, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9836956521739131, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0024919393061872295, "hi": 0.07252397558849175, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.8157894736842105, "hi": 0.8636363636363636, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.27801455917504586, "hi": 0.4241379484486101, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.331858407079646, "hi": 0.8597758958053077, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.08614528023598807, "hi": 1.3542600896860986, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.8130434782608695, "hi": 0.8652173913043478, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.2781521739130434, "hi": 0.43043478260869567, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.665929203539823, "hi": 0.9298879479026538, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.043072640117994034, "hi": 0.6771300448430493, "validReplicates": 1998, "degenerate": false } } } } }, { "first": "claude-sonnet-5-5", "members": [ "claude-sonnet-5-5", "gpt-6-astra", "gemini-4-argon-high", "grok-4.5", "claude-opus-5-5" ], "reused": true, "firstStage": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "panelOnly": { "hypotheses": 230, "questions": 46, "selected": 129, "matches": 7, "known": 7, "missed": 0, "counts": { "yes_true": 7, "yes_false": 122, "no_true": 0, "no_false": 101 }, "rates": { "ppv": 0.05426356589147287, "npv": 1, "sens": 1, "spec": 0.452914798206278, "youden": 0.452914798206278, "accuracy": 0.46956521739130436, "balancedAccuracy": 0.726457399103139 } }, "final": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "conditional": { "hypotheses": 45, "questions": 41, "selected": 45, "matches": 6, "known": 6, "missed": 0, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 0, "no_false": 0 }, "rates": { "ppv": 0.13333333333333333, "npv": null, "sens": 1, "spec": 0, "youden": 0, "accuracy": 0.13333333333333333, "balancedAccuracy": 0.5 } }, "coin": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "conditionalCoin": { "hypotheses": 45, "questions": 41, "selected": 23, "matches": 2, "known": 6, "missed": 4, "counts": { "yes_true": 2, "yes_false": 21, "no_true": 4, "no_false": 18 }, "rates": { "ppv": 0.08695652173913043, "npv": 0.8181818181818182, "sens": 0.3333333333333333, "spec": 0.46153846153846156, "youden": -0.20512820512820507, "accuracy": 0.4444444444444444, "balancedAccuracy": 0.39743589743589747 } }, "lostAtFirst": 1, "lostAtSecond": 0, "removedAtSecond": 0, "workload": { "first": 230, "survivors": 45, "additionalModels": 4, "second": 180, "total": 410, "fullPanel": 1150, "saved": 740 }, "selectedIds": [ "R42581103|H2", "R42605193|H4", "R42627902|H3", "R42630044|H3", "R42639736|H1", "R42642518|H5", "R42642521|H1", "R42647119|H1", "R42647119|H2", "R42647635|H1", "R42663085|H1", "R42664348|H3", "R42665751|H3", "R42689491|H1", "R42689491|H4", "R42696595|H3", "R42698274|H4", "R42717261|H5", "R42723546|H2", "R42744804|H3", "R42759510|H4", "R42775695|H4", "R42786310|H4", "R42805954|H2", "R42810392|H3", "R42810392|H5", "R42811497|H1", "R42821663|H1", "R42822442|H5", "R42826184|H1", "R42839094|H4", "R42841227|H3", "R42842666|H1", "R42844664|H3", "R42853938|H1", "R42858183|H1", "R42806945|H4", "R42808804|H1", "R42585340|H1", "R42669067|H1", "R42669067|H5", "R42811065|H1", "R42841610|H3", "R42806003|H1", "R42806572|H2" ], "firstLostIds": [ "R42842666|H4" ], "secondLostIds": [], "comparison": { "hypotheses": 230, "questions": 46, "left": { "hypotheses": 230, "questions": 46, "selected": 45, "matches": 6, "known": 7, "missed": 1, "counts": { "yes_true": 6, "yes_false": 39, "no_true": 1, "no_false": 184 }, "rates": { "ppv": 0.13333333333333333, "npv": 0.9945945945945946, "sens": 0.8571428571428571, "spec": 0.8251121076233184, "youden": 0.6822549647661755, "accuracy": 0.8260869565217391, "balancedAccuracy": 0.8411274823830878 } }, "right": { "hypotheses": 230, "questions": 46, "selected": 117, "matches": 3, "known": 7, "missed": 4, "counts": { "yes_true": 3, "yes_false": 114, "no_true": 4, "no_false": 109 }, "rates": { "ppv": 0.02564102564102564, "npv": 0.9646017699115044, "sens": 0.42857142857142855, "spec": 0.48878923766816146, "youden": -0.08263933376041, "accuracy": 0.48695652173913045, "balancedAccuracy": 0.458680333119795 } }, "differences": { "ppv": 0.1076923076923077, "npv": 0.02999282468309028, "sens": 0.42857142857142855, "spec": 0.336322869955157, "youden": 0.7648942985265855, "accuracy": 0.3391304347826087, "balancedAccuracy": 0.38244714926329276 }, "uncertainty": { "method": "paired original-question cluster percentile bootstrap", "replicates": 2000, "intervals": { "left_ppv": { "lo": 0.045454545454545456, "hi": 0.22920673076923054, "validReplicates": 2000, "degenerate": false }, "right_ppv": { "lo": 0, "hi": 0.05825444983818769, "validReplicates": 2000, "degenerate": false }, "difference_ppv": { "lo": 0.024375725900116146, "hi": 0.19610939592322565, "validReplicates": 2000, "degenerate": false }, "left_npv": { "lo": 0.9834254143646409, "hi": 1, "validReplicates": 2000, "degenerate": false }, "right_npv": { "lo": 0.9259259259259259, "hi": 0.9919370967741935, "validReplicates": 2000, "degenerate": false }, "difference_npv": { "lo": -0.0026588774428116854, "hi": 0.07216170080695225, "validReplicates": 2000, "degenerate": false }, "left_sens": { "lo": 0.5, "hi": 1, "validReplicates": 1998, "degenerate": false }, "right_sens": { "lo": 0, "hi": 0.8333333333333334, "validReplicates": 1998, "degenerate": false }, "difference_sens": { "lo": -0.25, "hi": 1, "validReplicates": 1998, "degenerate": false }, "left_spec": { "lo": 0.7981449298791577, "hi": 0.852017937219731, "validReplicates": 2000, "degenerate": false }, "right_spec": { "lo": 0.42342342342342343, "hi": 0.5520480795057748, "validReplicates": 2000, "degenerate": false }, "difference_spec": { "lo": 0.2678571428571428, "hi": 0.40708365034236976, "validReplicates": 2000, "degenerate": false }, "left_youden": { "lo": 0.31858407079646023, "hi": 0.8468468468468469, "validReplicates": 1998, "degenerate": false }, "right_youden": { "lo": -0.5204941336643574, "hi": 0.3686899423446511, "validReplicates": 1998, "degenerate": false }, "difference_youden": { "lo": 0.07382352590029291, "hi": 1.34376060520362, "validReplicates": 1998, "degenerate": false }, "left_accuracy": { "lo": 0.7956521739130434, "hi": 0.8521739130434782, "validReplicates": 2000, "degenerate": false }, "right_accuracy": { "lo": 0.41739130434782606, "hi": 0.5565217391304348, "validReplicates": 2000, "degenerate": false }, "difference_accuracy": { "lo": 0.26521739130434785, "hi": 0.4130434782608696, "validReplicates": 2000, "degenerate": false }, "left_balancedAccuracy": { "lo": 0.6592920353982301, "hi": 0.9234234234234234, "validReplicates": 1998, "degenerate": false }, "right_balancedAccuracy": { "lo": 0.23975293316782126, "hi": 0.6843449711723255, "validReplicates": 1998, "degenerate": false }, "difference_balancedAccuracy": { "lo": 0.036911762950146455, "hi": 0.67188030260181, "validReplicates": 1998, "degenerate": false } } } } } ] } ] }