{ "protocol": "experimental-support-coin-v1", "forecastProtocol": "experimental-support-v1", "inputSha256": "f1f53a813f414f6a2c30e6b75eaa96e770cf703ad93e32204b69bee8ee561039", "createdAt": "2026-10-22T16:34:16.758Z", "hypotheses": 250, "questions": 50, "heads": 117, "tails": 133, "metrics": [ { "endpoint": "strict", "policy": "advice", "validHypotheses": 250, "validQuestions": 50, "unknownProbabilities": 250, "selected": 117, "referenceSupported": 2, "selectedWithoutReferenceSupport": 115, "notSelected": 133, "allKnownPositives": 4, "knownPositivesInValidQuestions": 4, "missedKnownPositives": 2, "recoveredFractionOfAllKnown": 0.5, "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "uncertainty": { "method": "task-cluster percentile bootstrap", "tasks": 50, "replicates": 10000, "intervals": { "ppv": { "lo": 0, "hi": 0.04424778761061947, "valid_replicates": 10000 }, "npv": { "lo": 0.9621212121212122, "hi": 1, "valid_replicates": 10000 }, "sens": { "lo": 0, "hi": 1, "valid_replicates": 9824 }, "spec": { "lo": 0.476, "hi": 0.5870445344129555, "valid_replicates": 10000 } }, "degenerate": [], "interpretation": "Task-cluster descriptive intervals for article correspondence. Unmatched hypotheses remain biologically unlabelled; these are not intervals for future experimental success." } }, { "endpoint": "core", "policy": "advice", "validHypotheses": 250, "validQuestions": 50, "unknownProbabilities": 250, "selected": 117, "referenceSupported": 17, "selectedWithoutReferenceSupport": 100, "notSelected": 133, "allKnownPositives": 36, "knownPositivesInValidQuestions": 36, "missedKnownPositives": 19, "recoveredFractionOfAllKnown": 0.4722222222222222, "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "uncertainty": { "method": "task-cluster percentile bootstrap", "tasks": 50, "replicates": 10000, "intervals": { "ppv": { "lo": 0.09016393442622951, "hi": 0.20661869478483888, "valid_replicates": 10000 }, "npv": { "lo": 0.7839895522388061, "hi": 0.9241405433646812, "valid_replicates": 10000 }, "sens": { "lo": 0.34283834586466166, "hi": 0.6333333333333333, "valid_replicates": 10000 }, "spec": { "lo": 0.46918694301458086, "hi": 0.5954545454545455, "valid_replicates": 10000 } }, "degenerate": [], "interpretation": "Task-cluster descriptive intervals for article correspondence. Unmatched hypotheses remain biologically unlabelled; these are not intervals for future experimental success." } } ], "tosses": [ { "task": "R42581103", "id": "H1", "recommend": true }, { "task": "R42581103", "id": "H2", "recommend": true }, { "task": "R42581103", "id": "H3", "recommend": true }, { "task": "R42581103", "id": "H4", "recommend": false }, { "task": "R42581103", "id": "H5", "recommend": true }, { "task": "R42601407", "id": "H1", "recommend": false }, { "task": "R42601407", "id": "H2", "recommend": true }, { "task": "R42601407", "id": "H3", "recommend": true }, { "task": "R42601407", "id": "H4", "recommend": false }, { "task": "R42601407", "id": "H5", "recommend": true }, { "task": "R42605193", "id": "H1", "recommend": false }, { "task": "R42605193", "id": "H2", "recommend": true }, { "task": "R42605193", "id": "H3", "recommend": false }, { "task": "R42605193", "id": "H4", "recommend": true }, { "task": "R42605193", "id": "H5", "recommend": false }, { "task": "R42608398", "id": "H1", "recommend": false }, { "task": "R42608398", "id": "H2", "recommend": true }, { "task": "R42608398", "id": "H3", "recommend": true }, { "task": "R42608398", "id": "H4", "recommend": false }, { "task": "R42608398", "id": "H5", "recommend": false }, { "task": "R42627902", "id": "H1", "recommend": true }, { "task": "R42627902", "id": "H2", "recommend": false }, { "task": "R42627902", "id": "H3", "recommend": false }, { "task": "R42627902", "id": "H4", "recommend": false }, { "task": "R42627902", "id": "H5", "recommend": false }, { "task": "R42630044", "id": "H1", "recommend": false }, { "task": "R42630044", "id": "H2", "recommend": false }, { "task": "R42630044", "id": "H3", "recommend": false }, { "task": "R42630044", "id": "H4", "recommend": false }, { "task": "R42630044", "id": "H5", "recommend": true }, { "task": "R42639736", "id": "H1", "recommend": false }, { "task": "R42639736", "id": "H2", "recommend": false }, { "task": "R42639736", "id": "H3", "recommend": false }, { "task": "R42639736", "id": "H4", "recommend": true }, { "task": "R42639736", "id": "H5", "recommend": true }, { "task": "R42642518", "id": "H1", "recommend": true }, { "task": "R42642518", "id": "H2", "recommend": false }, { "task": "R42642518", "id": "H3", "recommend": false }, { "task": "R42642518", "id": "H4", "recommend": true }, { "task": "R42642518", "id": "H5", "recommend": false }, { "task": "R42642521", "id": "H1", "recommend": true }, { "task": "R42642521", "id": "H2", "recommend": false }, { "task": "R42642521", "id": "H3", "recommend": true }, { "task": "R42642521", "id": "H4", "recommend": false }, { "task": "R42642521", "id": "H5", "recommend": true }, { "task": "R42647119", "id": "H1", "recommend": false }, { "task": "R42647119", "id": "H2", "recommend": true }, { "task": "R42647119", "id": "H3", "recommend": true }, { "task": "R42647119", "id": "H4", "recommend": false }, { "task": "R42647119", "id": "H5", "recommend": false }, { "task": "R42647635", "id": "H1", "recommend": true }, { "task": "R42647635", "id": "H2", "recommend": false }, { "task": "R42647635", "id": "H3", "recommend": false }, { "task": "R42647635", "id": "H4", "recommend": true }, { "task": "R42647635", "id": "H5", "recommend": false }, { "task": "R42663085", "id": "H1", "recommend": true }, { "task": "R42663085", "id": "H2", "recommend": false }, { "task": "R42663085", "id": "H3", "recommend": false }, { "task": "R42663085", "id": "H4", "recommend": true }, { "task": "R42663085", "id": "H5", "recommend": false }, { "task": "R42664348", "id": "H1", "recommend": false }, { "task": "R42664348", "id": "H2", "recommend": false }, { "task": "R42664348", "id": "H3", "recommend": false }, { "task": "R42664348", "id": "H4", "recommend": true }, { "task": "R42664348", "id": "H5", "recommend": true }, { "task": "R42665751", "id": "H1", "recommend": true }, { "task": "R42665751", "id": "H2", "recommend": true }, { "task": "R42665751", "id": "H3", "recommend": false }, { "task": "R42665751", "id": "H4", "recommend": false }, { "task": "R42665751", "id": "H5", "recommend": true }, { "task": "R42689491", "id": "H1", "recommend": false }, { "task": "R42689491", "id": "H2", "recommend": true }, { "task": "R42689491", "id": "H3", "recommend": false }, { "task": "R42689491", "id": "H4", "recommend": false }, { "task": "R42689491", "id": "H5", "recommend": false }, { "task": "R42696595", "id": "H1", "recommend": true }, { "task": "R42696595", "id": "H2", "recommend": true }, { "task": "R42696595", "id": "H3", "recommend": true }, { "task": "R42696595", "id": "H4", "recommend": false }, { "task": "R42696595", "id": "H5", "recommend": true }, { "task": "R42698274", "id": "H1", "recommend": false }, { "task": "R42698274", "id": "H2", "recommend": false }, { "task": "R42698274", "id": "H3", "recommend": true }, { "task": "R42698274", "id": "H4", "recommend": true }, { "task": "R42698274", "id": "H5", "recommend": true }, { "task": "R42717261", "id": "H1", "recommend": false }, { "task": "R42717261", "id": "H2", "recommend": true }, { "task": "R42717261", "id": "H3", "recommend": false }, { "task": "R42717261", "id": "H4", "recommend": false }, { "task": "R42717261", "id": "H5", "recommend": true }, { "task": "R42723546", "id": "H1", "recommend": false }, { "task": "R42723546", "id": "H2", "recommend": true }, { "task": "R42723546", "id": "H3", "recommend": false }, { "task": "R42723546", "id": "H4", "recommend": false }, { "task": "R42723546", "id": "H5", "recommend": false }, { "task": "R42725837", "id": "H1", "recommend": true }, { "task": "R42725837", "id": "H2", "recommend": false }, { "task": "R42725837", "id": "H3", "recommend": false }, { "task": "R42725837", "id": "H4", "recommend": true }, { "task": "R42725837", "id": "H5", "recommend": true }, { "task": "R42736449", "id": "H1", "recommend": false }, { "task": "R42736449", "id": "H2", "recommend": true }, { "task": "R42736449", "id": "H3", "recommend": true }, { "task": "R42736449", "id": "H4", "recommend": true }, { "task": "R42736449", "id": "H5", "recommend": false }, { "task": "R42744804", "id": "H1", "recommend": false }, { "task": "R42744804", "id": "H2", "recommend": true }, { "task": "R42744804", "id": "H3", "recommend": false }, { "task": "R42744804", "id": "H4", "recommend": true }, { "task": "R42744804", "id": "H5", "recommend": true }, { "task": "R42749700", "id": "H1", "recommend": false }, { "task": "R42749700", "id": "H2", "recommend": false }, { "task": "R42749700", "id": "H3", "recommend": false }, { "task": "R42749700", "id": "H4", "recommend": false }, { "task": "R42749700", "id": "H5", "recommend": true }, { "task": "R42758848", "id": "H1", "recommend": false }, { "task": "R42758848", "id": "H2", "recommend": true }, { "task": "R42758848", "id": "H3", "recommend": false }, { "task": "R42758848", "id": "H4", "recommend": false }, { "task": "R42758848", "id": "H5", "recommend": false }, { "task": "R42759510", "id": "H1", "recommend": true }, { "task": "R42759510", "id": "H2", "recommend": true }, { "task": "R42759510", "id": "H3", "recommend": true }, { "task": "R42759510", "id": "H4", "recommend": true }, { "task": "R42759510", "id": "H5", "recommend": false }, { "task": "R42775695", "id": "H1", "recommend": false }, { "task": "R42775695", "id": "H2", "recommend": true }, { "task": "R42775695", "id": "H3", "recommend": false }, { "task": "R42775695", "id": "H4", "recommend": false }, { "task": "R42775695", "id": "H5", "recommend": false }, { "task": "R42775698", "id": "H1", "recommend": true }, { "task": "R42775698", "id": "H2", "recommend": false }, { "task": "R42775698", "id": "H3", "recommend": false }, { "task": "R42775698", "id": "H4", "recommend": false }, { "task": "R42775698", "id": "H5", "recommend": true }, { "task": "R42786310", "id": "H1", "recommend": true }, { "task": "R42786310", "id": "H2", "recommend": true }, { "task": "R42786310", "id": "H3", "recommend": false }, { "task": "R42786310", "id": "H4", "recommend": false }, { "task": "R42786310", "id": "H5", "recommend": false }, { "task": "R42805954", "id": "H1", "recommend": false }, { "task": "R42805954", "id": "H2", "recommend": false }, { "task": "R42805954", "id": "H3", "recommend": false }, { "task": "R42805954", "id": "H4", "recommend": false }, { "task": "R42805954", "id": "H5", "recommend": true }, { "task": "R42810392", "id": "H1", "recommend": true }, { "task": "R42810392", "id": "H2", "recommend": false }, { "task": "R42810392", "id": "H3", "recommend": false }, { "task": "R42810392", "id": "H4", "recommend": true }, { "task": "R42810392", "id": "H5", "recommend": false }, { "task": "R42811497", "id": "H1", "recommend": true }, { "task": "R42811497", "id": "H2", "recommend": true }, { "task": "R42811497", "id": "H3", "recommend": false }, { "task": "R42811497", "id": "H4", "recommend": true }, { "task": "R42811497", "id": "H5", "recommend": true }, { "task": "R42821663", "id": "H1", "recommend": false }, { "task": "R42821663", "id": "H2", "recommend": true }, { "task": "R42821663", "id": "H3", "recommend": false }, { "task": "R42821663", "id": "H4", "recommend": true }, { "task": "R42821663", "id": "H5", "recommend": false }, { "task": "R42822442", "id": "H1", "recommend": true }, { "task": "R42822442", "id": "H2", "recommend": false }, { "task": "R42822442", "id": "H3", "recommend": false }, { "task": "R42822442", "id": "H4", "recommend": true }, { "task": "R42822442", "id": "H5", "recommend": true }, { "task": "R42826184", "id": "H1", "recommend": false }, { "task": "R42826184", "id": "H2", "recommend": true }, { "task": "R42826184", "id": "H3", "recommend": false }, { "task": "R42826184", "id": "H4", "recommend": false }, { "task": "R42826184", "id": "H5", "recommend": true }, { "task": "R42839094", "id": "H1", "recommend": true }, { "task": "R42839094", "id": "H2", "recommend": true }, { "task": "R42839094", "id": "H3", "recommend": false }, { "task": "R42839094", "id": "H4", "recommend": true }, { "task": "R42839094", "id": "H5", "recommend": false }, { "task": "R42841227", "id": "H1", "recommend": true }, { "task": "R42841227", "id": "H2", "recommend": true }, { "task": "R42841227", "id": "H3", "recommend": true }, { "task": "R42841227", "id": "H4", "recommend": true }, { "task": "R42841227", "id": "H5", "recommend": false }, { "task": "R42842666", "id": "H1", "recommend": false }, { "task": "R42842666", "id": "H2", "recommend": false }, { "task": "R42842666", "id": "H3", "recommend": true }, { "task": "R42842666", "id": "H4", "recommend": true }, { "task": "R42842666", "id": "H5", "recommend": false }, { "task": "R42844664", "id": "H1", "recommend": true }, { "task": "R42844664", "id": "H2", "recommend": false }, { "task": "R42844664", "id": "H3", "recommend": true }, { "task": "R42844664", "id": "H4", "recommend": false }, { "task": "R42844664", "id": "H5", "recommend": true }, { "task": "R42853938", "id": "H1", "recommend": false }, { "task": "R42853938", "id": "H2", "recommend": false }, { "task": "R42853938", "id": "H3", "recommend": true }, { "task": "R42853938", "id": "H4", "recommend": false }, { "task": "R42853938", "id": "H5", "recommend": true }, { "task": "R42858183", "id": "H1", "recommend": true }, { "task": "R42858183", "id": "H2", "recommend": false }, { "task": "R42858183", "id": "H3", "recommend": true }, { "task": "R42858183", "id": "H4", "recommend": false }, { "task": "R42858183", "id": "H5", "recommend": false }, { "task": "R42806945", "id": "H1", "recommend": true }, { "task": "R42806945", "id": "H2", "recommend": true }, { "task": "R42806945", "id": "H3", "recommend": true }, { "task": "R42806945", "id": "H4", "recommend": false }, { "task": "R42806945", "id": "H5", "recommend": false }, { "task": "R42723144", "id": "H1", "recommend": true }, { "task": "R42723144", "id": "H2", "recommend": true }, { "task": "R42723144", "id": "H3", "recommend": false }, { "task": "R42723144", "id": "H4", "recommend": false }, { "task": "R42723144", "id": "H5", "recommend": true }, { "task": "R42808804", "id": "H1", "recommend": false }, { "task": "R42808804", "id": "H2", "recommend": false }, { "task": "R42808804", "id": "H3", "recommend": false }, { "task": "R42808804", "id": "H4", "recommend": true }, { "task": "R42808804", "id": "H5", "recommend": false }, { "task": "R42585340", "id": "H1", "recommend": false }, { "task": "R42585340", "id": "H2", "recommend": false }, { "task": "R42585340", "id": "H3", "recommend": true }, { "task": "R42585340", "id": "H4", "recommend": true }, { "task": "R42585340", "id": "H5", "recommend": false }, { "task": "R42669067", "id": "H1", "recommend": false }, { "task": "R42669067", "id": "H2", "recommend": false }, { "task": "R42669067", "id": "H3", "recommend": true }, { "task": "R42669067", "id": "H4", "recommend": false }, { "task": "R42669067", "id": "H5", "recommend": false }, { "task": "R42811065", "id": "H1", "recommend": false }, { "task": "R42811065", "id": "H2", "recommend": false }, { "task": "R42811065", "id": "H3", "recommend": true }, { "task": "R42811065", "id": "H4", "recommend": false }, { "task": "R42811065", "id": "H5", "recommend": false }, { "task": "R42841610", "id": "H1", "recommend": true }, { "task": "R42841610", "id": "H2", "recommend": true }, { "task": "R42841610", "id": "H3", "recommend": true }, { "task": "R42841610", "id": "H4", "recommend": false }, { "task": "R42841610", "id": "H5", "recommend": false }, { "task": "R42806003", "id": "H1", "recommend": true }, { "task": "R42806003", "id": "H2", "recommend": true }, { "task": "R42806003", "id": "H3", "recommend": false }, { "task": "R42806003", "id": "H4", "recommend": false }, { "task": "R42806003", "id": "H5", "recommend": true }, { "task": "R42806572", "id": "H1", "recommend": true }, { "task": "R42806572", "id": "H2", "recommend": true }, { "task": "R42806572", "id": "H3", "recommend": true }, { "task": "R42806572", "id": "H4", "recommend": true }, { "task": "R42806572", "id": "H5", "recommend": true }, { "task": "R42748238", "id": "H1", "recommend": true }, { "task": "R42748238", "id": "H2", "recommend": true }, { "task": "R42748238", "id": "H3", "recommend": false }, { "task": "R42748238", "id": "H4", "recommend": false }, { "task": "R42748238", "id": "H5", "recommend": true } ], "comparisons": [ { "modelId": "claude-sonnet-5-5", "endpoint": "strict", "policy": "advice", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 4, "yes_false": 124, "no_true": 0, "no_false": 122 }, "rates": { "ppv": 0.03125, "npv": 1, "sens": 1, "spec": 0.4959349593495935, "youden": 0.4959349593495934 }, "selected": 128, "referenceSupported": 4 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": 0.014155982905982904, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.0021681070429592324, "hi": 0.038461538461538464, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-sonnet-5-5", "endpoint": "strict", "policy": "p50", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 17, "no_true": 4, "no_false": 229 }, "rates": { "ppv": 0, "npv": 0.9828326180257511, "sens": 0, "spec": 0.9308943089430894, "youden": -0.06910569105691056 }, "selected": 17, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": -0.017094017094017096, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.04424778761061947, "hi": 0, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-sonnet-5-5", "endpoint": "strict", "policy": "p70", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 2, "no_true": 4, "no_false": 244 }, "rates": { "ppv": 0, "npv": 0.9838709677419355, "sens": 0, "spec": 0.991869918699187, "youden": -0.008130081300813052 }, "selected": 2, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": -0.017094017094017096, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.04386935258500232, "hi": 0, "valid_replicates": 8760 }, "degenerate": false } }, { "modelId": "claude-sonnet-5-5", "endpoint": "strict", "policy": "p90", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 4, "no_false": 246 }, "rates": { "ppv": null, "npv": 0.984, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "claude-sonnet-5-5", "endpoint": "core", "policy": "advice", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 26, "yes_false": 102, "no_true": 10, "no_false": 112 }, "rates": { "ppv": 0.203125, "npv": 0.9180327868852459, "sens": 0.7222222222222222, "spec": 0.5233644859813084, "youden": 0.24558670820353057 }, "selected": 128, "referenceSupported": 26 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.05782585470085469, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.001255378931865507, "hi": 0.12114426312467882, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-sonnet-5-5", "endpoint": "core", "policy": "p50", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 7, "yes_false": 10, "no_true": 29, "no_false": 204 }, "rates": { "ppv": 0.4117647058823529, "npv": 0.8755364806866953, "sens": 0.19444444444444445, "spec": 0.9532710280373832, "youden": 0.1477154724818277 }, "selected": 17, "referenceSupported": 7 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.2664655605832076, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": 0.02484280054807578, "hi": 0.5133865015196498, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-sonnet-5-5", "endpoint": "core", "policy": "p70", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 1, "yes_false": 1, "no_true": 35, "no_false": 213 }, "rates": { "ppv": 0.5, "npv": 0.8588709677419355, "sens": 0.027777777777777776, "spec": 0.9953271028037384, "youden": 0.023104880581516163 }, "selected": 2, "referenceSupported": 1 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.35470085470085466, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.19047619047619047, "hi": 0.9, "valid_replicates": 8760 }, "degenerate": false } }, { "modelId": "claude-sonnet-5-5", "endpoint": "core", "policy": "p90", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 36, "no_false": 214 }, "rates": { "ppv": null, "npv": 0.856, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "claude-opus-5-5", "endpoint": "strict", "policy": "advice", "hypotheses": 215, "questions": 43, "model": { "counts": { "yes_true": 4, "yes_false": 99, "no_true": 0, "no_false": 112 }, "rates": { "ppv": 0.038834951456310676, "npv": 1, "sens": 1, "spec": 0.5308056872037915, "youden": 0.5308056872037916 }, "selected": 103, "referenceSupported": 4 }, "coin": { "counts": { "yes_true": 2, "yes_false": 98, "no_true": 2, "no_false": 113 }, "rates": { "ppv": 0.02, "npv": 0.9826086956521739, "sens": 0.5, "spec": 0.5355450236966824, "youden": 0.03554502369668233 }, "selected": 100, "referenceSupported": 2 }, "ppvDifference": 0.018834951456310676, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.0015053763440860228, "hi": 0.049696975522795, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-opus-5-5", "endpoint": "strict", "policy": "p50", "hypotheses": 215, "questions": 43, "model": { "counts": { "yes_true": 2, "yes_false": 59, "no_true": 2, "no_false": 152 }, "rates": { "ppv": 0.03278688524590164, "npv": 0.987012987012987, "sens": 0.5, "spec": 0.7203791469194313, "youden": 0.22037914691943117 }, "selected": 61, "referenceSupported": 2 }, "coin": { "counts": { "yes_true": 2, "yes_false": 98, "no_true": 2, "no_false": 113 }, "rates": { "ppv": 0.02, "npv": 0.9826086956521739, "sens": 0.5, "spec": 0.5355450236966824, "youden": 0.03554502369668233 }, "selected": 100, "referenceSupported": 2 }, "ppvDifference": 0.01278688524590164, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.021052631578947368, "hi": 0.05882644102417082, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-opus-5-5", "endpoint": "strict", "policy": "p70", "hypotheses": 215, "questions": 43, "model": { "counts": { "yes_true": 2, "yes_false": 11, "no_true": 2, "no_false": 200 }, "rates": { "ppv": 0.15384615384615385, "npv": 0.9900990099009901, "sens": 0.5, "spec": 0.9478672985781991, "youden": 0.44786729857819907 }, "selected": 13, "referenceSupported": 2 }, "coin": { "counts": { "yes_true": 2, "yes_false": 98, "no_true": 2, "no_false": 113 }, "rates": { "ppv": 0.02, "npv": 0.9826086956521739, "sens": 0.5, "spec": 0.5355450236966824, "youden": 0.03554502369668233 }, "selected": 100, "referenceSupported": 2 }, "ppvDifference": 0.13384615384615386, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.019417475728155338, "hi": 0.3485893546607831, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-opus-5-5", "endpoint": "strict", "policy": "p90", "hypotheses": 215, "questions": 43, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 4, "no_false": 211 }, "rates": { "ppv": null, "npv": 0.9813953488372092, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 98, "no_true": 2, "no_false": 113 }, "rates": { "ppv": 0.02, "npv": 0.9826086956521739, "sens": 0.5, "spec": 0.5355450236966824, "youden": 0.03554502369668233 }, "selected": 100, "referenceSupported": 2 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "claude-opus-5-5", "endpoint": "core", "policy": "advice", "hypotheses": 215, "questions": 43, "model": { "counts": { "yes_true": 22, "yes_false": 81, "no_true": 7, "no_false": 105 }, "rates": { "ppv": 0.21359223300970873, "npv": 0.9375, "sens": 0.7586206896551724, "spec": 0.5645161290322581, "youden": 0.3231368186874306 }, "selected": 103, "referenceSupported": 22 }, "coin": { "counts": { "yes_true": 14, "yes_false": 86, "no_true": 15, "no_false": 100 }, "rates": { "ppv": 0.14, "npv": 0.8695652173913043, "sens": 0.4827586206896552, "spec": 0.5376344086021505, "youden": 0.020393029291805753 }, "selected": 100, "referenceSupported": 14 }, "ppvDifference": 0.07359223300970871, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.0011707043266008233, "hi": 0.15158432663654714, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-opus-5-5", "endpoint": "core", "policy": "p50", "hypotheses": 215, "questions": 43, "model": { "counts": { "yes_true": 16, "yes_false": 45, "no_true": 13, "no_false": 141 }, "rates": { "ppv": 0.26229508196721313, "npv": 0.9155844155844156, "sens": 0.5517241379310345, "spec": 0.7580645161290323, "youden": 0.3097886540600667 }, "selected": 61, "referenceSupported": 16 }, "coin": { "counts": { "yes_true": 14, "yes_false": 86, "no_true": 15, "no_false": 100 }, "rates": { "ppv": 0.14, "npv": 0.8695652173913043, "sens": 0.4827586206896552, "spec": 0.5376344086021505, "youden": 0.020393029291805753 }, "selected": 100, "referenceSupported": 14 }, "ppvDifference": 0.12229508196721312, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": 0.029807126811349678, "hi": 0.2211545950155763, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-opus-5-5", "endpoint": "core", "policy": "p70", "hypotheses": 215, "questions": 43, "model": { "counts": { "yes_true": 5, "yes_false": 8, "no_true": 24, "no_false": 178 }, "rates": { "ppv": 0.38461538461538464, "npv": 0.8811881188118812, "sens": 0.1724137931034483, "spec": 0.956989247311828, "youden": 0.1294030404152764 }, "selected": 13, "referenceSupported": 5 }, "coin": { "counts": { "yes_true": 14, "yes_false": 86, "no_true": 15, "no_false": 100 }, "rates": { "ppv": 0.14, "npv": 0.8695652173913043, "sens": 0.4827586206896552, "spec": 0.5376344086021505, "youden": 0.020393029291805753 }, "selected": 100, "referenceSupported": 14 }, "ppvDifference": 0.24461538461538462, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.013427715695157523, "hi": 0.5162713524099661, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-opus-5-5", "endpoint": "core", "policy": "p90", "hypotheses": 215, "questions": 43, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 29, "no_false": 186 }, "rates": { "ppv": null, "npv": 0.8651162790697674, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 14, "yes_false": 86, "no_true": 15, "no_false": 100 }, "rates": { "ppv": 0.14, "npv": 0.8695652173913043, "sens": 0.4827586206896552, "spec": 0.5376344086021505, "youden": 0.020393029291805753 }, "selected": 100, "referenceSupported": 14 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "claude-haiku-5-5", "endpoint": "strict", "policy": "advice", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 3, "yes_false": 106, "no_true": 1, "no_false": 140 }, "rates": { "ppv": 0.027522935779816515, "npv": 0.9929078014184397, "sens": 0.75, "spec": 0.5691056910569106, "youden": 0.31910569105691056 }, "selected": 109, "referenceSupported": 3 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": 0.01042891868579942, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.000866039597657247, "hi": 0.030288025889967634, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-haiku-5-5", "endpoint": "strict", "policy": "p50", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 12, "no_true": 4, "no_false": 234 }, "rates": { "ppv": 0, "npv": 0.9831932773109243, "sens": 0, "spec": 0.9512195121951219, "youden": -0.04878048780487809 }, "selected": 12, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": -0.017094017094017096, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.04424778761061947, "hi": 0, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-haiku-5-5", "endpoint": "strict", "policy": "p70", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 4, "no_false": 246 }, "rates": { "ppv": null, "npv": 0.984, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "claude-haiku-5-5", "endpoint": "strict", "policy": "p90", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 4, "no_false": 246 }, "rates": { "ppv": null, "npv": 0.984, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "claude-haiku-5-5", "endpoint": "core", "policy": "advice", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 20, "yes_false": 89, "no_true": 16, "no_false": 125 }, "rates": { "ppv": 0.1834862385321101, "npv": 0.8865248226950354, "sens": 0.5555555555555556, "spec": 0.5841121495327103, "youden": 0.13966770508826576 }, "selected": 109, "referenceSupported": 20 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.03818709323296479, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.04379300458715596, "hi": 0.12480526224684094, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-haiku-5-5", "endpoint": "core", "policy": "p50", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 5, "yes_false": 7, "no_true": 31, "no_false": 207 }, "rates": { "ppv": 0.4166666666666667, "npv": 0.8697478991596639, "sens": 0.1388888888888889, "spec": 0.9672897196261683, "youden": 0.10617860851505712 }, "selected": 12, "referenceSupported": 5 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.2713675213675214, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.031316672624012075, "hi": 0.5823403426791274, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-haiku-5-5", "endpoint": "core", "policy": "p70", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 36, "no_false": 214 }, "rates": { "ppv": null, "npv": 0.856, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "claude-haiku-5-5", "endpoint": "core", "policy": "p90", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 36, "no_false": 214 }, "rates": { "ppv": null, "npv": 0.856, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "claude-opus-4-8", "endpoint": "strict", "policy": "advice", "hypotheses": 240, "questions": 48, "model": { "counts": { "yes_true": 3, "yes_false": 131, "no_true": 1, "no_false": 105 }, "rates": { "ppv": 0.022388059701492536, "npv": 0.9905660377358491, "sens": 0.75, "spec": 0.4449152542372881, "youden": 0.19491525423728806 }, "selected": 134, "referenceSupported": 3 }, "coin": { "counts": { "yes_true": 2, "yes_false": 110, "no_true": 2, "no_false": 126 }, "rates": { "ppv": 0.017857142857142856, "npv": 0.984375, "sens": 0.5, "spec": 0.5338983050847458, "youden": 0.03389830508474567 }, "selected": 112, "referenceSupported": 2 }, "ppvDifference": 0.00453091684434968, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.022431077694235586, "hi": 0.03091398651614443, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-opus-4-8", "endpoint": "strict", "policy": "p50", "hypotheses": 240, "questions": 48, "model": { "counts": { "yes_true": 1, "yes_false": 77, "no_true": 3, "no_false": 159 }, "rates": { "ppv": 0.01282051282051282, "npv": 0.9814814814814815, "sens": 0.25, "spec": 0.673728813559322, "youden": -0.07627118644067798 }, "selected": 78, "referenceSupported": 1 }, "coin": { "counts": { "yes_true": 2, "yes_false": 110, "no_true": 2, "no_false": 126 }, "rates": { "ppv": 0.017857142857142856, "npv": 0.984375, "sens": 0.5, "spec": 0.5338983050847458, "youden": 0.03389830508474567 }, "selected": 112, "referenceSupported": 2 }, "ppvDifference": -0.005036630036630036, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.02727272727272727, "hi": 0.010466988727858294, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-opus-4-8", "endpoint": "strict", "policy": "p70", "hypotheses": 240, "questions": 48, "model": { "counts": { "yes_true": 1, "yes_false": 4, "no_true": 3, "no_false": 232 }, "rates": { "ppv": 0.2, "npv": 0.9872340425531915, "sens": 0.25, "spec": 0.9830508474576272, "youden": 0.23305084745762716 }, "selected": 5, "referenceSupported": 1 }, "coin": { "counts": { "yes_true": 2, "yes_false": 110, "no_true": 2, "no_false": 126 }, "rates": { "ppv": 0.017857142857142856, "npv": 0.984375, "sens": 0.5, "spec": 0.5338983050847458, "youden": 0.03389830508474567 }, "selected": 112, "referenceSupported": 2 }, "ppvDifference": 0.18214285714285716, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.02586206896551724, "hi": 0.6361239950525666, "valid_replicates": 9952 }, "degenerate": false } }, { "modelId": "claude-opus-4-8", "endpoint": "strict", "policy": "p90", "hypotheses": 240, "questions": 48, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 4, "no_false": 236 }, "rates": { "ppv": null, "npv": 0.9833333333333333, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 110, "no_true": 2, "no_false": 126 }, "rates": { "ppv": 0.017857142857142856, "npv": 0.984375, "sens": 0.5, "spec": 0.5338983050847458, "youden": 0.03389830508474567 }, "selected": 112, "referenceSupported": 2 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "claude-opus-4-8", "endpoint": "core", "policy": "advice", "hypotheses": 240, "questions": 48, "model": { "counts": { "yes_true": 28, "yes_false": 106, "no_true": 6, "no_false": 100 }, "rates": { "ppv": 0.208955223880597, "npv": 0.9433962264150944, "sens": 0.8235294117647058, "spec": 0.4854368932038835, "youden": 0.30896630496858934 }, "selected": 134, "referenceSupported": 28 }, "coin": { "counts": { "yes_true": 16, "yes_false": 96, "no_true": 18, "no_false": 110 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.859375, "sens": 0.47058823529411764, "spec": 0.5339805825242718, "youden": 0.004568817818389359 }, "selected": 112, "referenceSupported": 16 }, "ppvDifference": 0.06609808102345416, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.00045132808709044594, "hi": 0.1326132799864956, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-opus-4-8", "endpoint": "core", "policy": "p50", "hypotheses": 240, "questions": 48, "model": { "counts": { "yes_true": 18, "yes_false": 60, "no_true": 16, "no_false": 146 }, "rates": { "ppv": 0.23076923076923078, "npv": 0.9012345679012346, "sens": 0.5294117647058824, "spec": 0.7087378640776699, "youden": 0.23814962878355228 }, "selected": 78, "referenceSupported": 18 }, "coin": { "counts": { "yes_true": 16, "yes_false": 96, "no_true": 18, "no_false": 110 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.859375, "sens": 0.47058823529411764, "spec": 0.5339805825242718, "youden": 0.004568817818389359 }, "selected": 112, "referenceSupported": 16 }, "ppvDifference": 0.08791208791208793, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.00430750934420661, "hi": 0.18181954600021824, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "claude-opus-4-8", "endpoint": "core", "policy": "p70", "hypotheses": 240, "questions": 48, "model": { "counts": { "yes_true": 4, "yes_false": 1, "no_true": 30, "no_false": 205 }, "rates": { "ppv": 0.8, "npv": 0.8723404255319149, "sens": 0.11764705882352941, "spec": 0.9951456310679612, "youden": 0.1127926898914906 }, "selected": 5, "referenceSupported": 4 }, "coin": { "counts": { "yes_true": 16, "yes_false": 96, "no_true": 18, "no_false": 110 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.859375, "sens": 0.47058823529411764, "spec": 0.5339805825242718, "youden": 0.004568817818389359 }, "selected": 112, "referenceSupported": 16 }, "ppvDifference": 0.6571428571428573, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": 0.17724438239595797, "hi": 0.905982905982906, "valid_replicates": 9952 }, "degenerate": false } }, { "modelId": "claude-opus-4-8", "endpoint": "core", "policy": "p90", "hypotheses": 240, "questions": 48, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 34, "no_false": 206 }, "rates": { "ppv": null, "npv": 0.8583333333333333, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 16, "yes_false": 96, "no_true": 18, "no_false": 110 }, "rates": { "ppv": 0.14285714285714285, "npv": 0.859375, "sens": 0.47058823529411764, "spec": 0.5339805825242718, "youden": 0.004568817818389359 }, "selected": 112, "referenceSupported": 16 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "gpt-6-astra", "endpoint": "strict", "policy": "advice", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 3, "yes_false": 197, "no_true": 1, "no_false": 49 }, "rates": { "ppv": 0.015, "npv": 0.98, "sens": 0.75, "spec": 0.1991869918699187, "youden": -0.05081300813008127 }, "selected": 200, "referenceSupported": 3 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": -0.0020940170940170963, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.025510828438950986, "hi": 0.0196078431372549, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gpt-6-astra", "endpoint": "strict", "policy": "p50", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 3, "yes_false": 98, "no_true": 1, "no_false": 148 }, "rates": { "ppv": 0.0297029702970297, "npv": 0.9932885906040269, "sens": 0.75, "spec": 0.6016260162601627, "youden": 0.35162601626016254 }, "selected": 101, "referenceSupported": 3 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": 0.012608953203012606, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.01679234851835471, "hi": 0.0469603740923834, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gpt-6-astra", "endpoint": "strict", "policy": "p70", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 1, "yes_false": 18, "no_true": 3, "no_false": 228 }, "rates": { "ppv": 0.05263157894736842, "npv": 0.987012987012987, "sens": 0.25, "spec": 0.926829268292683, "youden": 0.17682926829268286 }, "selected": 19, "referenceSupported": 1 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": 0.03553756185335132, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.03636363636363636, "hi": 0.14989473684210525, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gpt-6-astra", "endpoint": "strict", "policy": "p90", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 4, "no_false": 246 }, "rates": { "ppv": null, "npv": 0.984, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "gpt-6-astra", "endpoint": "core", "policy": "advice", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 32, "yes_false": 168, "no_true": 4, "no_false": 46 }, "rates": { "ppv": 0.16, "npv": 0.92, "sens": 0.8888888888888888, "spec": 0.21495327102803738, "youden": 0.10384215991692614 }, "selected": 200, "referenceSupported": 32 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.014700854700854693, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.03789021422522496, "hi": 0.06573225331101824, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gpt-6-astra", "endpoint": "core", "policy": "p50", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 22, "yes_false": 79, "no_true": 14, "no_false": 135 }, "rates": { "ppv": 0.21782178217821782, "npv": 0.9060402684563759, "sens": 0.6111111111111112, "spec": 0.6308411214953271, "youden": 0.24195223260643828 }, "selected": 101, "referenceSupported": 22 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.07252263687907251, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.0021964968348825386, "hi": 0.15136966335914315, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gpt-6-astra", "endpoint": "core", "policy": "p70", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 8, "yes_false": 11, "no_true": 28, "no_false": 203 }, "rates": { "ppv": 0.42105263157894735, "npv": 0.8787878787878788, "sens": 0.2222222222222222, "spec": 0.9485981308411215, "youden": 0.17082035306334387 }, "selected": 19, "referenceSupported": 8 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.275753486279802, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.012196616451458642, "hi": 0.5612068872746838, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gpt-6-astra", "endpoint": "core", "policy": "p90", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 36, "no_false": 214 }, "rates": { "ppv": null, "npv": 0.856, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "gemini-4-argon-high", "endpoint": "strict", "policy": "advice", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 4, "yes_false": 148, "no_true": 0, "no_false": 98 }, "rates": { "ppv": 0.02631578947368421, "npv": 1, "sens": 1, "spec": 0.3983739837398374, "youden": 0.39837398373983746 }, "selected": 152, "referenceSupported": 4 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": 0.009221772379667113, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.006705064595447246, "hi": 0.03134023512382655, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gemini-4-argon-high", "endpoint": "strict", "policy": "p50", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 3, "yes_false": 131, "no_true": 1, "no_false": 115 }, "rates": { "ppv": 0.022388059701492536, "npv": 0.9913793103448276, "sens": 0.75, "spec": 0.46747967479674796, "youden": 0.217479674796748 }, "selected": 134, "referenceSupported": 3 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": 0.00529404260747544, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.02000561389651981, "hi": 0.0329265264147617, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gemini-4-argon-high", "endpoint": "strict", "policy": "p70", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 3, "yes_false": 37, "no_true": 1, "no_false": 209 }, "rates": { "ppv": 0.075, "npv": 0.9952380952380953, "sens": 0.75, "spec": 0.8495934959349594, "youden": 0.5995934959349594 }, "selected": 40, "referenceSupported": 3 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": 0.0579059829059829, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.008547008547008548, "hi": 0.14740238317031512, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gemini-4-argon-high", "endpoint": "strict", "policy": "p90", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 4, "no_false": 246 }, "rates": { "ppv": null, "npv": 0.984, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 115, "no_true": 2, "no_false": 131 }, "rates": { "ppv": 0.017094017094017096, "npv": 0.9849624060150376, "sens": 0.5, "spec": 0.532520325203252, "youden": 0.03252032520325199 }, "selected": 117, "referenceSupported": 2 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "gemini-4-argon-high", "endpoint": "core", "policy": "advice", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 32, "yes_false": 120, "no_true": 4, "no_false": 94 }, "rates": { "ppv": 0.21052631578947367, "npv": 0.9591836734693877, "sens": 0.8888888888888888, "spec": 0.4392523364485981, "youden": 0.3281412253374869 }, "selected": 152, "referenceSupported": 32 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.06522717049032836, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": 0.0010312813313839166, "hi": 0.13219833259393626, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gemini-4-argon-high", "endpoint": "core", "policy": "p50", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 31, "yes_false": 103, "no_true": 5, "no_false": 111 }, "rates": { "ppv": 0.23134328358208955, "npv": 0.9568965517241379, "sens": 0.8611111111111112, "spec": 0.5186915887850467, "youden": 0.3798026998961579 }, "selected": 134, "referenceSupported": 31 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.08604413828294424, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": 0.01443516627295785, "hi": 0.1639715264203931, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gemini-4-argon-high", "endpoint": "core", "policy": "p70", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 14, "yes_false": 26, "no_true": 22, "no_false": 188 }, "rates": { "ppv": 0.35, "npv": 0.8952380952380953, "sens": 0.3888888888888889, "spec": 0.8785046728971962, "youden": 0.2673935617860852 }, "selected": 40, "referenceSupported": 14 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": 0.20470085470085467, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": 0.05207417978614855, "hi": 0.3699460144927534, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "gemini-4-argon-high", "endpoint": "core", "policy": "p90", "hypotheses": 250, "questions": 50, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 36, "no_false": 214 }, "rates": { "ppv": null, "npv": 0.856, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 17, "yes_false": 100, "no_true": 19, "no_false": 114 }, "rates": { "ppv": 0.1452991452991453, "npv": 0.8571428571428571, "sens": 0.4722222222222222, "spec": 0.5327102803738317, "youden": 0.004932502596053823 }, "selected": 117, "referenceSupported": 17 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "grok-4.5", "endpoint": "strict", "policy": "advice", "hypotheses": 225, "questions": 45, "model": { "counts": { "yes_true": 3, "yes_false": 134, "no_true": 0, "no_false": 88 }, "rates": { "ppv": 0.021897810218978103, "npv": 1, "sens": 1, "spec": 0.3963963963963964, "youden": 0.39639639639639634 }, "selected": 137, "referenceSupported": 3 }, "coin": { "counts": { "yes_true": 2, "yes_false": 107, "no_true": 1, "no_false": 115 }, "rates": { "ppv": 0.01834862385321101, "npv": 0.9913793103448276, "sens": 0.6666666666666666, "spec": 0.5180180180180181, "youden": 0.1846846846846848 }, "selected": 109, "referenceSupported": 2 }, "ppvDifference": 0.003549186365767093, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.009137503685684797, "hi": 0.02054794520547945, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "grok-4.5", "endpoint": "strict", "policy": "p50", "hypotheses": 225, "questions": 45, "model": { "counts": { "yes_true": 2, "yes_false": 45, "no_true": 1, "no_false": 177 }, "rates": { "ppv": 0.0425531914893617, "npv": 0.9943820224719101, "sens": 0.6666666666666666, "spec": 0.7972972972972973, "youden": 0.4639639639639639 }, "selected": 47, "referenceSupported": 2 }, "coin": { "counts": { "yes_true": 2, "yes_false": 107, "no_true": 1, "no_false": 115 }, "rates": { "ppv": 0.01834862385321101, "npv": 0.9913793103448276, "sens": 0.6666666666666666, "spec": 0.5180180180180181, "youden": 0.1846846846846848 }, "selected": 109, "referenceSupported": 2 }, "ppvDifference": 0.02420456763615069, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.01834862385321101, "hi": 0.08239019660894654, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "grok-4.5", "endpoint": "strict", "policy": "p70", "hypotheses": 225, "questions": 45, "model": { "counts": { "yes_true": 0, "yes_false": 6, "no_true": 3, "no_false": 216 }, "rates": { "ppv": 0, "npv": 0.9863013698630136, "sens": 0, "spec": 0.972972972972973, "youden": -0.027027027027026973 }, "selected": 6, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 107, "no_true": 1, "no_false": 115 }, "rates": { "ppv": 0.01834862385321101, "npv": 0.9913793103448276, "sens": 0.6666666666666666, "spec": 0.5180180180180181, "youden": 0.1846846846846848 }, "selected": 109, "referenceSupported": 2 }, "ppvDifference": -0.01834862385321101, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.047619047619047616, "hi": 0, "valid_replicates": 9983 }, "degenerate": false } }, { "modelId": "grok-4.5", "endpoint": "strict", "policy": "p90", "hypotheses": 225, "questions": 45, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 3, "no_false": 222 }, "rates": { "ppv": null, "npv": 0.9866666666666667, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 2, "yes_false": 107, "no_true": 1, "no_false": 115 }, "rates": { "ppv": 0.01834862385321101, "npv": 0.9913793103448276, "sens": 0.6666666666666666, "spec": 0.5180180180180181, "youden": 0.1846846846846848 }, "selected": 109, "referenceSupported": 2 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } }, { "modelId": "grok-4.5", "endpoint": "core", "policy": "advice", "hypotheses": 225, "questions": 45, "model": { "counts": { "yes_true": 27, "yes_false": 110, "no_true": 4, "no_false": 84 }, "rates": { "ppv": 0.19708029197080293, "npv": 0.9545454545454546, "sens": 0.8709677419354839, "spec": 0.4329896907216495, "youden": 0.30395743265713326 }, "selected": 137, "referenceSupported": 27 }, "coin": { "counts": { "yes_true": 16, "yes_false": 93, "no_true": 15, "no_false": 101 }, "rates": { "ppv": 0.14678899082568808, "npv": 0.8706896551724138, "sens": 0.5161290322580645, "spec": 0.520618556701031, "youden": 0.036747588959095356 }, "selected": 109, "referenceSupported": 16 }, "ppvDifference": 0.05029130114511485, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.016435348125055687, "hi": 0.11723679561914854, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "grok-4.5", "endpoint": "core", "policy": "p50", "hypotheses": 225, "questions": 45, "model": { "counts": { "yes_true": 12, "yes_false": 35, "no_true": 19, "no_false": 159 }, "rates": { "ppv": 0.2553191489361702, "npv": 0.8932584269662921, "sens": 0.3870967741935484, "spec": 0.8195876288659794, "youden": 0.20668440305952762 }, "selected": 47, "referenceSupported": 12 }, "coin": { "counts": { "yes_true": 16, "yes_false": 93, "no_true": 15, "no_false": 101 }, "rates": { "ppv": 0.14678899082568808, "npv": 0.8706896551724138, "sens": 0.5161290322580645, "spec": 0.520618556701031, "youden": 0.036747588959095356 }, "selected": 109, "referenceSupported": 16 }, "ppvDifference": 0.10853015811048211, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.007118122354751816, "hi": 0.233115871645442, "valid_replicates": 10000 }, "degenerate": false } }, { "modelId": "grok-4.5", "endpoint": "core", "policy": "p70", "hypotheses": 225, "questions": 45, "model": { "counts": { "yes_true": 2, "yes_false": 4, "no_true": 29, "no_false": 190 }, "rates": { "ppv": 0.3333333333333333, "npv": 0.867579908675799, "sens": 0.06451612903225806, "spec": 0.979381443298969, "youden": 0.04389757233122715 }, "selected": 6, "referenceSupported": 2 }, "coin": { "counts": { "yes_true": 16, "yes_false": 93, "no_true": 15, "no_false": 101 }, "rates": { "ppv": 0.14678899082568808, "npv": 0.8706896551724138, "sens": 0.5161290322580645, "spec": 0.520618556701031, "youden": 0.036747588959095356 }, "selected": 109, "referenceSupported": 16 }, "ppvDifference": 0.18654434250764523, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": { "lo": -0.16822429906542055, "hi": 0.6545454545454545, "valid_replicates": 9983 }, "degenerate": false } }, { "modelId": "grok-4.5", "endpoint": "core", "policy": "p90", "hypotheses": 225, "questions": 45, "model": { "counts": { "yes_true": 0, "yes_false": 0, "no_true": 31, "no_false": 194 }, "rates": { "ppv": null, "npv": 0.8622222222222222, "sens": 0, "spec": 1, "youden": 0 }, "selected": 0, "referenceSupported": 0 }, "coin": { "counts": { "yes_true": 16, "yes_false": 93, "no_true": 15, "no_false": 101 }, "rates": { "ppv": 0.14678899082568808, "npv": 0.8706896551724138, "sens": 0.5161290322580645, "spec": 0.520618556701031, "youden": 0.036747588959095356 }, "selected": 109, "referenceSupported": 16 }, "ppvDifference": null, "uncertainty": { "method": "task-cluster percentile bootstrap", "replicates": 10000, "interval": null, "degenerate": false } } ], "design": { "protocol": "experimental-support-coin-v1", "createdAt": "2026-10-22T16:32:49.172774+00:00", "forecastProtocol": "experimental-support-v1", "inputSha256": "f1f53a813f414f6a2c30e6b75eaa96e770cf703ad93e32204b69bee8ee561039", "questions": 50, "hypotheses": 250, "drawAlgorithm": "Node crypto.randomBytes(32); first 250 bits, most-significant bit first in frozen source task/candidate order; final six bits unused", "heads": "recommend testing", "tails": "do not recommend testing", "redraws": 0, "timing": "Coin added after model results were inspected; draw generation uses only fixed hypothesis identities and random bytes, never article labels or model decisions.", "pairedScope": "exact valid hypothesis identities for each model, jointly resampled by original question", "replicates": 10000, "sources": { "summaries/forecast-research.json": "21a22b4bfedc50059563127add79c48dc40de86663cfb44c7d81934f05f9b966", "summaries/experimental-support-summary.json": "e04da325083c75557bba271e897203f496d4758f078c2ba58d83f9f8c85b0cb1", "https://eternalsearch.net/omega/experimental-support-20261022/claude-sonnet-5-5.json": "745c4ca36b272de4f24362b7d5c1858d32bd589f3851e7fddd5177d958bf80f9", "https://eternalsearch.net/omega/experimental-support-20261022/claude-opus-5-5.json": "40c3b17065ab1ab5a2f808ada20f8b1fa37057872958be07125527e9456f46c5", "https://eternalsearch.net/omega/experimental-support-20261022/claude-haiku-5-5.json": "6a3750d3ead03a0cae4fc6bd292967586cbdf72932579b9dd70b6b7f1d98b076", "https://eternalsearch.net/omega/experimental-support-20261022/claude-opus-4-8.json": "f918bb5282262b13ff3f3a9491204da73c8179c3883a9507abcb1a56cf20df5c", "https://eternalsearch.net/omega/experimental-support-20261022/gpt-6-astra.json": "c695752c0e1b011f1d39dc11925582234176ab4c755ee7b9ca68eaf9b07f0597", "https://eternalsearch.net/omega/experimental-support-20261022/gemini-4-argon-high.json": "7e8f33deff79491444c3e3150c8d2f851a107b955e1aff29e485d2bdc8087000", "https://eternalsearch.net/omega/experimental-support-20261022/grok-4.5.json": "5625fd8a3935d5ed2bf5b9f7cdf57ccc8e9a4a8a95d816e5b27836650f518d89" } }, "draw": { "protocol": "experimental-support-coin-v1", "protocolSha256": "a06f3be759049d9bb8b12ed75f5d08203790d4684c349708f4b410725dc913ef", "createdAt": "2026-10-22T16:34:16.758Z", "randomBytesHex": "eb54c80472ab2521e51d3a51372c28f22380cb6a9a75e354b4e644621399fe68", "tosses": [ { "task": "R42581103", "id": "H1", "recommend": true }, { "task": "R42581103", "id": "H2", "recommend": true }, { "task": "R42581103", "id": "H3", "recommend": true }, { "task": "R42581103", "id": "H4", "recommend": false }, { "task": "R42581103", "id": "H5", "recommend": true }, { "task": "R42601407", "id": "H1", "recommend": false }, { "task": "R42601407", "id": "H2", "recommend": true }, { "task": "R42601407", "id": "H3", "recommend": true }, { "task": "R42601407", "id": "H4", "recommend": false }, { "task": "R42601407", "id": "H5", "recommend": true }, { "task": "R42605193", "id": "H1", "recommend": false }, { "task": "R42605193", "id": "H2", "recommend": true }, { "task": "R42605193", "id": "H3", "recommend": false }, { "task": "R42605193", "id": "H4", "recommend": true }, { "task": "R42605193", "id": "H5", "recommend": false }, { "task": "R42608398", "id": "H1", "recommend": false }, { "task": "R42608398", "id": "H2", "recommend": true }, { "task": "R42608398", "id": "H3", "recommend": true }, { "task": "R42608398", "id": "H4", "recommend": false }, { "task": "R42608398", "id": "H5", "recommend": false }, { "task": "R42627902", "id": "H1", "recommend": true }, { "task": "R42627902", "id": "H2", "recommend": false }, { "task": "R42627902", "id": "H3", "recommend": false }, { "task": "R42627902", "id": "H4", "recommend": false }, { "task": "R42627902", "id": "H5", "recommend": false }, { "task": "R42630044", "id": "H1", "recommend": false }, { "task": "R42630044", "id": "H2", "recommend": false }, { "task": "R42630044", "id": "H3", "recommend": false }, { "task": "R42630044", "id": "H4", "recommend": false }, { "task": "R42630044", "id": "H5", "recommend": true }, { "task": "R42639736", "id": "H1", "recommend": false }, { "task": "R42639736", "id": "H2", "recommend": false }, { "task": "R42639736", "id": "H3", "recommend": false }, { "task": "R42639736", "id": "H4", "recommend": true }, { "task": "R42639736", "id": "H5", "recommend": true }, { "task": "R42642518", "id": "H1", "recommend": true }, { "task": "R42642518", "id": "H2", "recommend": false }, { "task": "R42642518", "id": "H3", "recommend": false }, { "task": "R42642518", "id": "H4", "recommend": true }, { "task": "R42642518", "id": "H5", "recommend": false }, { "task": "R42642521", "id": "H1", "recommend": true }, { "task": "R42642521", "id": "H2", "recommend": false }, { "task": "R42642521", "id": "H3", "recommend": true }, { "task": "R42642521", "id": "H4", "recommend": false }, { "task": "R42642521", "id": "H5", "recommend": true }, { "task": "R42647119", "id": "H1", "recommend": false }, { "task": "R42647119", "id": "H2", "recommend": true }, { "task": "R42647119", "id": "H3", "recommend": true }, { "task": "R42647119", "id": "H4", "recommend": false }, { "task": "R42647119", "id": "H5", "recommend": false }, { "task": "R42647635", "id": "H1", "recommend": true }, { "task": "R42647635", "id": "H2", "recommend": false }, { "task": "R42647635", "id": "H3", "recommend": false }, { "task": "R42647635", "id": "H4", "recommend": true }, { "task": "R42647635", "id": "H5", "recommend": false }, { "task": "R42663085", "id": "H1", "recommend": true }, { "task": "R42663085", "id": "H2", "recommend": false }, { "task": "R42663085", "id": "H3", "recommend": false }, { "task": "R42663085", "id": "H4", "recommend": true }, { "task": "R42663085", "id": "H5", "recommend": false }, { "task": "R42664348", "id": "H1", "recommend": false }, { "task": "R42664348", "id": "H2", "recommend": false }, { "task": "R42664348", "id": "H3", "recommend": false }, { "task": "R42664348", "id": "H4", "recommend": true }, { "task": "R42664348", "id": "H5", "recommend": true }, { "task": "R42665751", "id": "H1", "recommend": true }, { "task": "R42665751", "id": "H2", "recommend": true }, { "task": "R42665751", "id": "H3", "recommend": false }, { "task": "R42665751", "id": "H4", "recommend": false }, { "task": "R42665751", "id": "H5", "recommend": true }, { "task": "R42689491", "id": "H1", "recommend": false }, { "task": "R42689491", "id": "H2", "recommend": true }, { "task": "R42689491", "id": "H3", "recommend": false }, { "task": "R42689491", "id": "H4", "recommend": false }, { "task": "R42689491", "id": "H5", "recommend": false }, { "task": "R42696595", "id": "H1", "recommend": true }, { "task": "R42696595", "id": "H2", "recommend": true }, { "task": "R42696595", "id": "H3", "recommend": true }, { "task": "R42696595", "id": "H4", "recommend": false }, { "task": "R42696595", "id": "H5", "recommend": true }, { "task": "R42698274", "id": "H1", "recommend": false }, { "task": "R42698274", "id": "H2", "recommend": false }, { "task": "R42698274", "id": "H3", "recommend": true }, { "task": "R42698274", "id": "H4", "recommend": true }, { "task": "R42698274", "id": "H5", "recommend": true }, { "task": "R42717261", "id": "H1", "recommend": false }, { "task": "R42717261", "id": "H2", "recommend": true }, { "task": "R42717261", "id": "H3", "recommend": false }, { "task": "R42717261", "id": "H4", "recommend": false }, { "task": "R42717261", "id": "H5", "recommend": true }, { "task": "R42723546", "id": "H1", "recommend": false }, { "task": "R42723546", "id": "H2", "recommend": true }, { "task": "R42723546", "id": "H3", "recommend": false }, { "task": "R42723546", "id": "H4", "recommend": false }, { "task": "R42723546", "id": "H5", "recommend": false }, { "task": "R42725837", "id": "H1", "recommend": true }, { "task": "R42725837", "id": "H2", "recommend": false }, { "task": "R42725837", "id": "H3", "recommend": false }, { "task": "R42725837", "id": "H4", "recommend": true }, { "task": "R42725837", "id": "H5", "recommend": true }, { "task": "R42736449", "id": "H1", "recommend": false }, { "task": "R42736449", "id": "H2", "recommend": true }, { "task": "R42736449", "id": "H3", "recommend": true }, { "task": "R42736449", "id": "H4", "recommend": true }, { "task": "R42736449", "id": "H5", "recommend": false }, { "task": "R42744804", "id": "H1", "recommend": false }, { "task": "R42744804", "id": "H2", "recommend": true }, { "task": "R42744804", "id": "H3", "recommend": false }, { "task": "R42744804", "id": "H4", "recommend": true }, { "task": "R42744804", "id": "H5", "recommend": true }, { "task": "R42749700", "id": "H1", "recommend": false }, { "task": "R42749700", "id": "H2", "recommend": false }, { "task": "R42749700", "id": "H3", "recommend": false }, { "task": "R42749700", "id": "H4", "recommend": false }, { "task": "R42749700", "id": "H5", "recommend": true }, { "task": "R42758848", "id": "H1", "recommend": false }, { "task": "R42758848", "id": "H2", "recommend": true }, { "task": "R42758848", "id": "H3", "recommend": false }, { "task": "R42758848", "id": "H4", "recommend": false }, { "task": "R42758848", "id": "H5", "recommend": false }, { "task": "R42759510", "id": "H1", "recommend": true }, { "task": "R42759510", "id": "H2", "recommend": true }, { "task": "R42759510", "id": "H3", "recommend": true }, { "task": "R42759510", "id": "H4", "recommend": true }, { "task": "R42759510", "id": "H5", "recommend": false }, { "task": "R42775695", "id": "H1", "recommend": false }, { "task": "R42775695", "id": "H2", "recommend": true }, { "task": "R42775695", "id": "H3", "recommend": false }, { "task": "R42775695", "id": "H4", "recommend": false }, { "task": "R42775695", "id": "H5", "recommend": false }, { "task": "R42775698", "id": "H1", "recommend": true }, { "task": "R42775698", "id": "H2", "recommend": false }, { "task": "R42775698", "id": "H3", "recommend": false }, { "task": "R42775698", "id": "H4", "recommend": false }, { "task": "R42775698", "id": "H5", "recommend": true }, { "task": "R42786310", "id": "H1", "recommend": true }, { "task": "R42786310", "id": "H2", "recommend": true }, { "task": "R42786310", "id": "H3", "recommend": false }, { "task": "R42786310", "id": "H4", "recommend": false }, { "task": "R42786310", "id": "H5", "recommend": false }, { "task": "R42805954", "id": "H1", "recommend": false }, { "task": "R42805954", "id": "H2", "recommend": false }, { "task": "R42805954", "id": "H3", "recommend": false }, { "task": "R42805954", "id": "H4", "recommend": false }, { "task": "R42805954", "id": "H5", "recommend": true }, { "task": "R42810392", "id": "H1", "recommend": true }, { "task": "R42810392", "id": "H2", "recommend": false }, { "task": "R42810392", "id": "H3", "recommend": false }, { "task": "R42810392", "id": "H4", "recommend": true }, { "task": "R42810392", "id": "H5", "recommend": false }, { "task": "R42811497", "id": "H1", "recommend": true }, { "task": "R42811497", "id": "H2", "recommend": true }, { "task": "R42811497", "id": "H3", "recommend": false }, { "task": "R42811497", "id": "H4", "recommend": true }, { "task": "R42811497", "id": "H5", "recommend": true }, { "task": "R42821663", "id": "H1", "recommend": false }, { "task": "R42821663", "id": "H2", "recommend": true }, { "task": "R42821663", "id": "H3", "recommend": false }, { "task": "R42821663", "id": "H4", "recommend": true }, { "task": "R42821663", "id": "H5", "recommend": false }, { "task": "R42822442", "id": "H1", "recommend": true }, { "task": "R42822442", "id": "H2", "recommend": false }, { "task": "R42822442", "id": "H3", "recommend": false }, { "task": "R42822442", "id": "H4", "recommend": true }, { "task": "R42822442", "id": "H5", "recommend": true }, { "task": "R42826184", "id": "H1", "recommend": false }, { "task": "R42826184", "id": "H2", "recommend": true }, { "task": "R42826184", "id": "H3", "recommend": false }, { "task": "R42826184", "id": "H4", "recommend": false }, { "task": "R42826184", "id": "H5", "recommend": true }, { "task": "R42839094", "id": "H1", "recommend": true }, { "task": "R42839094", "id": "H2", "recommend": true }, { "task": "R42839094", "id": "H3", "recommend": false }, { "task": "R42839094", "id": "H4", "recommend": true }, { "task": "R42839094", "id": "H5", "recommend": false }, { "task": "R42841227", "id": "H1", "recommend": true }, { "task": "R42841227", "id": "H2", "recommend": true }, { "task": "R42841227", "id": "H3", "recommend": true }, { "task": "R42841227", "id": "H4", "recommend": true }, { "task": "R42841227", "id": "H5", "recommend": false }, { "task": "R42842666", "id": "H1", "recommend": false }, { "task": "R42842666", "id": "H2", "recommend": false }, { "task": "R42842666", "id": "H3", "recommend": true }, { "task": "R42842666", "id": "H4", "recommend": true }, { "task": "R42842666", "id": "H5", "recommend": false }, { "task": "R42844664", "id": "H1", "recommend": true }, { "task": "R42844664", "id": "H2", "recommend": false }, { "task": "R42844664", "id": "H3", "recommend": true }, { "task": "R42844664", "id": "H4", "recommend": false }, { "task": "R42844664", "id": "H5", "recommend": true }, { "task": "R42853938", "id": "H1", "recommend": false }, { "task": "R42853938", "id": "H2", "recommend": false }, { "task": "R42853938", "id": "H3", "recommend": true }, { "task": "R42853938", "id": "H4", "recommend": false }, { "task": "R42853938", "id": "H5", "recommend": true }, { "task": "R42858183", "id": "H1", "recommend": true }, { "task": "R42858183", "id": "H2", "recommend": false }, { "task": "R42858183", "id": "H3", "recommend": true }, { "task": "R42858183", "id": "H4", "recommend": false }, { "task": "R42858183", "id": "H5", "recommend": false }, { "task": "R42806945", "id": "H1", "recommend": true }, { "task": "R42806945", "id": "H2", "recommend": true }, { "task": "R42806945", "id": "H3", "recommend": true }, { "task": "R42806945", "id": "H4", "recommend": false }, { "task": "R42806945", "id": "H5", "recommend": false }, { "task": "R42723144", "id": "H1", "recommend": true }, { "task": "R42723144", "id": "H2", "recommend": true }, { "task": "R42723144", "id": "H3", "recommend": false }, { "task": "R42723144", "id": "H4", "recommend": false }, { "task": "R42723144", "id": "H5", "recommend": true }, { "task": "R42808804", "id": "H1", "recommend": false }, { "task": "R42808804", "id": "H2", "recommend": false }, { "task": "R42808804", "id": "H3", "recommend": false }, { "task": "R42808804", "id": "H4", "recommend": true }, { "task": "R42808804", "id": "H5", "recommend": false }, { "task": "R42585340", "id": "H1", "recommend": false }, { "task": "R42585340", "id": "H2", "recommend": false }, { "task": "R42585340", "id": "H3", "recommend": true }, { "task": "R42585340", "id": "H4", "recommend": true }, { "task": "R42585340", "id": "H5", "recommend": false }, { "task": "R42669067", "id": "H1", "recommend": false }, { "task": "R42669067", "id": "H2", "recommend": false }, { "task": "R42669067", "id": "H3", "recommend": true }, { "task": "R42669067", "id": "H4", "recommend": false }, { "task": "R42669067", "id": "H5", "recommend": false }, { "task": "R42811065", "id": "H1", "recommend": false }, { "task": "R42811065", "id": "H2", "recommend": false }, { "task": "R42811065", "id": "H3", "recommend": true }, { "task": "R42811065", "id": "H4", "recommend": false }, { "task": "R42811065", "id": "H5", "recommend": false }, { "task": "R42841610", "id": "H1", "recommend": true }, { "task": "R42841610", "id": "H2", "recommend": true }, { "task": "R42841610", "id": "H3", "recommend": true }, { "task": "R42841610", "id": "H4", "recommend": false }, { "task": "R42841610", "id": "H5", "recommend": false }, { "task": "R42806003", "id": "H1", "recommend": true }, { "task": "R42806003", "id": "H2", "recommend": true }, { "task": "R42806003", "id": "H3", "recommend": false }, { "task": "R42806003", "id": "H4", "recommend": false }, { "task": "R42806003", "id": "H5", "recommend": true }, { "task": "R42806572", "id": "H1", "recommend": true }, { "task": "R42806572", "id": "H2", "recommend": true }, { "task": "R42806572", "id": "H3", "recommend": true }, { "task": "R42806572", "id": "H4", "recommend": true }, { "task": "R42806572", "id": "H5", "recommend": true }, { "task": "R42748238", "id": "H1", "recommend": true }, { "task": "R42748238", "id": "H2", "recommend": true }, { "task": "R42748238", "id": "H3", "recommend": false }, { "task": "R42748238", "id": "H4", "recommend": false }, { "task": "R42748238", "id": "H5", "recommend": true } ] }, "drawSha256": "63ccb838ab31e1e560712e072c8840a6818a646dc4c4e580a4024f2e8d041983", "interpretation": "One fair binary recommendation per hypothesis; no biological probability assigned. Labels measure correspondence to a saved article, not future experimental truth. Same recorded coin on both endpoints and all policies. Paired PPV comparisons exclude technical missing model forecasts from both sides, with original questions jointly resampled 10000 times. Descriptive intervals condition on this one recorded coin; they do not integrate uncertainty across new toss sequences or test adjusted multiple-comparison significance.", "reproduction": "Saved model outputs are in reconstruction.zip; coin-reconstruction.zip adds this design, the recorded draw, the addendum and the frozen public inputs. Analysis code is available on request." }