{
  "method": {
    "seed": 20261027,
    "bootstrap_samples": 10000,
    "permutations": 99999,
    "unit": "task",
    "predictor_fields_used": false,
    "interval": "percentile bootstrap of whole tasks with ranks recalculated",
    "p": "(1 + permutations with absolute rho >= observed)/(P + 1)"
  },
  "provenance": {
    "annotation_commit": "82123e40",
    "sha256": {
      "reports/question-generation-association-2026-10-27/annotations.tsv": "13e0831afd697782a49767de746e2d85684e7ef2ff98084d8ce5d976ff8ccf19",
      "reports/question-generation-association-2026-10-27/tasks-for-annotation.json": "5ed4fd0338ac283bc795a591b6412e6a87a3438bc18cb61d7487cd04bd5a1c00",
      "papers/2026-10-25-benchmark-data/generation-outcomes.json": "b67424c82c8f1a3bc981301d906606c64d271929516b80178a8b9cb9b05d9ddc",
      "papers/2026-10-25-benchmark-data/judge-scores.json": "1d37234f5d5e73b09c7b42a16b6718722be32089e6d5ec6cc966daca30e468b1"
    }
  },
  "audit": {
    "tasks": 50,
    "hypotheses": 2997,
    "independently_rebuilt_outcomes": 2997,
    "core": 358,
    "hit": 57,
    "cells": [
      {
        "model": "gpt-6-astra",
        "arm": "N",
        "hypotheses": 250
      },
      {
        "model": "gpt-6-astra",
        "arm": "A",
        "hypotheses": 250
      },
      {
        "model": "gpt-6-astra",
        "arm": "O",
        "hypotheses": 250
      },
      {
        "model": "grok-4.5",
        "arm": "N",
        "hypotheses": 250
      },
      {
        "model": "grok-4.5",
        "arm": "A",
        "hypotheses": 250
      },
      {
        "model": "grok-4.5",
        "arm": "O",
        "hypotheses": 250
      },
      {
        "model": "claude-opus-5-5",
        "arm": "N",
        "hypotheses": 250
      },
      {
        "model": "claude-opus-5-5",
        "arm": "A",
        "hypotheses": 250
      },
      {
        "model": "claude-opus-5-5",
        "arm": "O",
        "hypotheses": 247
      },
      {
        "model": "gemini-4-argon-high",
        "arm": "N",
        "hypotheses": 250
      },
      {
        "model": "gemini-4-argon-high",
        "arm": "A",
        "hypotheses": 250
      },
      {
        "model": "gemini-4-argon-high",
        "arm": "O",
        "hypotheses": 250
      }
    ]
  },
  "regimes": {
    "N": {
      "core": {
        "n_tasks": 50,
        "rho": 0.16850312045705002,
        "ci95": [
          -0.15254959005704463,
          0.45874832795230175
        ],
        "p_permutation": 0.24318,
        "bootstrap_finite": 10000,
        "groups": [
          {
            "score": 0,
            "n_tasks": 8,
            "successes": 28,
            "hypotheses": 160,
            "task_mean": 0.175,
            "ci95": [
              0.08750000000000001,
              0.26666666666666666
            ]
          },
          {
            "score": 1,
            "n_tasks": 30,
            "successes": 66,
            "hypotheses": 600,
            "task_mean": 0.10999999999999997,
            "ci95": [
              0.07333333333333332,
              0.15000000000000002
            ]
          },
          {
            "score": 2,
            "n_tasks": 12,
            "successes": 62,
            "hypotheses": 240,
            "task_mean": 0.25833333333333336,
            "ci95": [
              0.1468638392857143,
              0.3681818181818182
            ]
          }
        ],
        "difference": {
          "high_minus_lower": 0.13464912280701757,
          "ci95": [
            0.020785621147463285,
            0.2500033875338753
          ]
        },
        "p_holm_primary_two": 0.43354
      },
      "hit": {
        "n_tasks": 50,
        "rho": 0.17458662615989567,
        "ci95": [
          -0.1567362794716281,
          0.47693742020807334
        ],
        "p_permutation": 0.21677,
        "bootstrap_finite": 10000,
        "groups": [
          {
            "score": 0,
            "n_tasks": 8,
            "successes": 4,
            "hypotheses": 160,
            "task_mean": 0.025,
            "ci95": [
              0.0,
              0.06818181818181819
            ]
          },
          {
            "score": 1,
            "n_tasks": 30,
            "successes": 9,
            "hypotheses": 600,
            "task_mean": 0.015000000000000001,
            "ci95": [
              0.004545454545454546,
              0.027419354838709678
            ]
          },
          {
            "score": 2,
            "n_tasks": 12,
            "successes": 12,
            "hypotheses": 240,
            "task_mean": 0.05000000000000001,
            "ci95": [
              0.01363636363636364,
              0.09000000000000001
            ]
          }
        ],
        "difference": {
          "high_minus_lower": 0.03289473684210527,
          "ci95": [
            -0.005477952280277858,
            0.07380457380457381
          ]
        },
        "p_holm_primary_two": 0.43354
      }
    },
    "A": {
      "core": {
        "n_tasks": 50,
        "rho": 0.21095208011346694,
        "ci95": [
          -0.07533368420028545,
          0.47287565217742067
        ],
        "p_permutation": 0.1424,
        "bootstrap_finite": 10000,
        "groups": [
          {
            "score": 0,
            "n_tasks": 8,
            "successes": 25,
            "hypotheses": 160,
            "task_mean": 0.15625000000000003,
            "ci95": [
              0.041666666666666664,
              0.3
            ]
          },
          {
            "score": 1,
            "n_tasks": 30,
            "successes": 72,
            "hypotheses": 600,
            "task_mean": 0.11999999999999998,
            "ci95": [
              0.06481481481481483,
              0.18666666666666665
            ]
          },
          {
            "score": 2,
            "n_tasks": 12,
            "successes": 61,
            "hypotheses": 240,
            "task_mean": 0.25416666666666665,
            "ci95": [
              0.13499038461538462,
              0.37778645833333324
            ]
          }
        ],
        "difference": {
          "high_minus_lower": 0.12653508771929825,
          "ci95": [
            -0.00474441563352513,
            0.2583354989604989
          ]
        }
      },
      "hit": {
        "n_tasks": 50,
        "rho": 0.3340950500571122,
        "ci95": [
          -0.0015507303785783169,
          0.6116286724752328
        ],
        "p_permutation": 0.01637,
        "bootstrap_finite": 10000,
        "groups": [
          {
            "score": 0,
            "n_tasks": 8,
            "successes": 3,
            "hypotheses": 160,
            "task_mean": 0.018750000000000003,
            "ci95": [
              0.0,
              0.06000000000000001
            ]
          },
          {
            "score": 1,
            "n_tasks": 30,
            "successes": 8,
            "hypotheses": 600,
            "task_mean": 0.013333333333333334,
            "ci95": [
              0.0,
              0.036669117647058797
            ]
          },
          {
            "score": 2,
            "n_tasks": 12,
            "successes": 13,
            "hypotheses": 240,
            "task_mean": 0.054166666666666675,
            "ci95": [
              0.014695378151260513,
              0.1
            ]
          }
        ],
        "difference": {
          "high_minus_lower": 0.039692982456140356,
          "ci95": [
            -0.003223737129987129,
            0.08772037173352962
          ]
        }
      }
    },
    "O": {
      "core": {
        "n_tasks": 50,
        "rho": -0.12931274448401292,
        "ci95": [
          -0.40445891371250836,
          0.16975250437163739
        ],
        "p_permutation": 0.37079,
        "bootstrap_finite": 10000,
        "groups": [
          {
            "score": 0,
            "n_tasks": 8,
            "successes": 10,
            "hypotheses": 160,
            "task_mean": 0.0625,
            "ci95": [
              0.01993750000000002,
              0.10833333333333335
            ]
          },
          {
            "score": 1,
            "n_tasks": 30,
            "successes": 26,
            "hypotheses": 597,
            "task_mean": 0.043333333333333335,
            "ci95": [
              0.019696969696969702,
              0.0732142857142857
            ]
          },
          {
            "score": 2,
            "n_tasks": 12,
            "successes": 8,
            "hypotheses": 240,
            "task_mean": 0.03333333333333333,
            "ci95": [
              0.01,
              0.06
            ]
          }
        ],
        "difference": {
          "high_minus_lower": -0.014035087719298255,
          "ci95": [
            -0.04854469854469855,
            0.019651842060378576
          ]
        }
      },
      "hit": {
        "n_tasks": 50,
        "rho": -0.04209082171721753,
        "ci95": [
          -0.33721343422245637,
          0.2827797976013246
        ],
        "p_permutation": 0.87072,
        "bootstrap_finite": 9852,
        "groups": [
          {
            "score": 0,
            "n_tasks": 8,
            "successes": 2,
            "hypotheses": 160,
            "task_mean": 0.0125,
            "ci95": [
              0.0,
              0.04
            ]
          },
          {
            "score": 1,
            "n_tasks": 30,
            "successes": 5,
            "hypotheses": 597,
            "task_mean": 0.008333333333333333,
            "ci95": [
              0.0,
              0.024074074074074074
            ]
          },
          {
            "score": 2,
            "n_tasks": 12,
            "successes": 1,
            "hypotheses": 240,
            "task_mean": 0.004166666666666667,
            "ci95": [
              0.0,
              0.014285714285714287
            ]
          }
        ],
        "difference": {
          "high_minus_lower": -0.005043859649122806,
          "ci95": [
            -0.02023809523809524,
            0.008835758835758838
          ]
        }
      }
    },
    "equal_all_regimes": {
      "core": {
        "n_tasks": 50,
        "rho": 0.16465331381918755,
        "ci95": [
          -0.13824241160048126,
          0.445987365835854
        ],
        "p_permutation": 0.25344,
        "bootstrap_finite": 10000,
        "groups": [
          {
            "score": 0,
            "n_tasks": 8,
            "successes": 63,
            "hypotheses": 480,
            "task_mean": 0.13124999999999998,
            "ci95": [
              0.052083333333333336,
              0.22000000000000003
            ]
          },
          {
            "score": 1,
            "n_tasks": 30,
            "successes": 164,
            "hypotheses": 1797,
            "task_mean": 0.09111111111111109,
            "ci95": [
              0.056171296296296296,
              0.13103564771668216
            ]
          },
          {
            "score": 2,
            "n_tasks": 12,
            "successes": 131,
            "hypotheses": 720,
            "task_mean": 0.18194444444444446,
            "ci95": [
              0.10277777777777779,
              0.2631011904761904
            ]
          }
        ],
        "difference": {
          "high_minus_lower": 0.08238304093567256,
          "ci95": [
            -0.0038401241570555204,
            0.17041907051282051
          ]
        }
      },
      "hit": {
        "n_tasks": 50,
        "rho": 0.254082730808732,
        "ci95": [
          -0.06238451361184365,
          0.542963828772458
        ],
        "p_permutation": 0.07284,
        "bootstrap_finite": 10000,
        "groups": [
          {
            "score": 0,
            "n_tasks": 8,
            "successes": 9,
            "hypotheses": 480,
            "task_mean": 0.01875,
            "ci95": [
              0.0,
              0.049999999999999996
            ]
          },
          {
            "score": 1,
            "n_tasks": 30,
            "successes": 22,
            "hypotheses": 1797,
            "task_mean": 0.012222222222222221,
            "ci95": [
              0.0022222222222222222,
              0.027941176470588233
            ]
          },
          {
            "score": 2,
            "n_tasks": 12,
            "successes": 26,
            "hypotheses": 720,
            "task_mean": 0.03611111111111111,
            "ci95": [
              0.01111111111111111,
              0.06389254385964908
            ]
          }
        ],
        "difference": {
          "high_minus_lower": 0.02251461988304093,
          "ci95": [
            -0.00541675135501355,
            0.05227295274170274
          ]
        }
      }
    }
  },
  "cells": [
    {
      "model": "gpt-6-astra",
      "arm": "N",
      "threshold": "core",
      "n_tasks": 50,
      "rho": 0.12609957920609993,
      "ci95": [
        -0.2086592473299703,
        0.43463854239742894
      ],
      "p_permutation": 0.38167,
      "bootstrap_finite": 10000,
      "p_holm_24": 1.0
    },
    {
      "model": "gpt-6-astra",
      "arm": "N",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": -0.035012723601449525,
      "ci95": [
        -0.3334156683557903,
        0.27221738284509905
      ],
      "p_permutation": 0.99195,
      "bootstrap_finite": 9841,
      "p_holm_24": 1.0
    },
    {
      "model": "gpt-6-astra",
      "arm": "A",
      "threshold": "core",
      "n_tasks": 50,
      "rho": 0.20472865249205363,
      "ci95": [
        -0.0725166682105396,
        0.4742834052230913
      ],
      "p_permutation": 0.15199,
      "bootstrap_finite": 10000,
      "p_holm_24": 1.0
    },
    {
      "model": "gpt-6-astra",
      "arm": "A",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": 0.09999200095987203,
      "ci95": [
        -0.08326465241794123,
        0.33200794703733155
      ],
      "p_permutation": 0.64652,
      "bootstrap_finite": 9562,
      "p_holm_24": 1.0
    },
    {
      "model": "gpt-6-astra",
      "arm": "O",
      "threshold": "core",
      "n_tasks": 50,
      "rho": -0.036763359781521995,
      "ci95": [
        -0.2977762058964673,
        0.23473034575082452
      ],
      "p_permutation": 0.98164,
      "bootstrap_finite": 9988,
      "p_holm_24": 1.0
    },
    {
      "model": "gpt-6-astra",
      "arm": "O",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": 0.13391894120102568,
      "ci95": [
        -0.0641483217481997,
        0.37240929172002385
      ],
      "p_permutation": 0.56495,
      "bootstrap_finite": 8670,
      "p_holm_24": 1.0
    },
    {
      "model": "grok-4.5",
      "arm": "N",
      "threshold": "core",
      "n_tasks": 50,
      "rho": 0.1307975186982843,
      "ci95": [
        -0.19136815018386047,
        0.43579382044325315
      ],
      "p_permutation": 0.36298,
      "bootstrap_finite": 10000,
      "p_holm_24": 1.0
    },
    {
      "model": "grok-4.5",
      "arm": "N",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": 0.041062257123883374,
      "ci95": [
        -0.24557995508707914,
        0.3260773253630123
      ],
      "p_permutation": 0.92614,
      "bootstrap_finite": 9999,
      "p_holm_24": 1.0
    },
    {
      "model": "grok-4.5",
      "arm": "A",
      "threshold": "core",
      "n_tasks": 50,
      "rho": 0.17411483078420242,
      "ci95": [
        -0.1321396218226868,
        0.45947040139417367
      ],
      "p_permutation": 0.22594,
      "bootstrap_finite": 10000,
      "p_holm_24": 1.0
    },
    {
      "model": "grok-4.5",
      "arm": "A",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": 0.22894561309678318,
      "ci95": [
        -0.14689855824698722,
        0.5319604433643331
      ],
      "p_permutation": 0.12326,
      "bootstrap_finite": 9999,
      "p_holm_24": 1.0
    },
    {
      "model": "grok-4.5",
      "arm": "O",
      "threshold": "core",
      "n_tasks": 50,
      "rho": -0.14900259186531473,
      "ci95": [
        -0.33611953075753653,
        0.016624996045592047
      ],
      "p_permutation": 0.34326,
      "bootstrap_finite": 9976,
      "p_holm_24": 1.0
    },
    {
      "model": "grok-4.5",
      "arm": "O",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": null,
      "ci95": [
        null,
        null
      ],
      "p_permutation": null,
      "bootstrap_finite": 0,
      "reason": "Correlation undefined because one variable is constant",
      "p_holm_24": null
    },
    {
      "model": "claude-opus-5-5",
      "arm": "N",
      "threshold": "core",
      "n_tasks": 50,
      "rho": 0.1678007085125573,
      "ci95": [
        -0.1320925176262093,
        0.44016242251699855
      ],
      "p_permutation": 0.24361,
      "bootstrap_finite": 10000,
      "p_holm_24": 1.0
    },
    {
      "model": "claude-opus-5-5",
      "arm": "N",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": 0.3905026785644705,
      "ci95": [
        0.12819182132583498,
        0.6002126814304184
      ],
      "p_permutation": 0.00834,
      "bootstrap_finite": 9960,
      "p_holm_24": 0.20016
    },
    {
      "model": "claude-opus-5-5",
      "arm": "A",
      "threshold": "core",
      "n_tasks": 50,
      "rho": 0.14510296108676596,
      "ci95": [
        -0.1931358373569169,
        0.4680829574670859
      ],
      "p_permutation": 0.31823,
      "bootstrap_finite": 10000,
      "p_holm_24": 1.0
    },
    {
      "model": "claude-opus-5-5",
      "arm": "A",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": 0.13391894120102568,
      "ci95": [
        -0.06099375455928332,
        0.3659007250201489
      ],
      "p_permutation": 0.56775,
      "bootstrap_finite": 8723,
      "p_holm_24": 1.0
    },
    {
      "model": "claude-opus-5-5",
      "arm": "O",
      "threshold": "core",
      "n_tasks": 50,
      "rho": -0.22512794745477346,
      "ci95": [
        -0.5263740715773283,
        0.13718280705624425
      ],
      "p_permutation": 0.11887,
      "bootstrap_finite": 9999,
      "p_holm_24": 1.0
    },
    {
      "model": "claude-opus-5-5",
      "arm": "O",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": null,
      "ci95": [
        null,
        null
      ],
      "p_permutation": null,
      "bootstrap_finite": 0,
      "reason": "Correlation undefined because one variable is constant",
      "p_holm_24": null
    },
    {
      "model": "gemini-4-argon-high",
      "arm": "N",
      "threshold": "core",
      "n_tasks": 50,
      "rho": 0.1433007420848836,
      "ci95": [
        -0.16187495141614344,
        0.4173522880216943
      ],
      "p_permutation": 0.31864,
      "bootstrap_finite": 10000,
      "p_holm_24": 1.0
    },
    {
      "model": "gemini-4-argon-high",
      "arm": "N",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": 0.13985311464040834,
      "ci95": [
        -0.241781261430967,
        0.48266282905971164
      ],
      "p_permutation": 0.34202,
      "bootstrap_finite": 9997,
      "p_holm_24": 1.0
    },
    {
      "model": "gemini-4-argon-high",
      "arm": "A",
      "threshold": "core",
      "n_tasks": 50,
      "rho": 0.17467757847231227,
      "ci95": [
        -0.13562982074150767,
        0.46191426788604534
      ],
      "p_permutation": 0.22694,
      "bootstrap_finite": 10000,
      "p_holm_24": 1.0
    },
    {
      "model": "gemini-4-argon-high",
      "arm": "A",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": 0.25734351847065395,
      "ci95": [
        -0.1277615061328246,
        0.5571044239757892
      ],
      "p_permutation": 0.05261,
      "bootstrap_finite": 9983,
      "p_holm_24": 1.0
    },
    {
      "model": "gemini-4-argon-high",
      "arm": "O",
      "threshold": "core",
      "n_tasks": 50,
      "rho": -0.017204153329987128,
      "ci95": [
        -0.2875375922368809,
        0.2585792117392264
      ],
      "p_permutation": 0.91759,
      "bootstrap_finite": 10000,
      "p_holm_24": 1.0
    },
    {
      "model": "gemini-4-argon-high",
      "arm": "O",
      "threshold": "hit",
      "n_tasks": 50,
      "rho": -0.16911451730461594,
      "ci95": [
        -0.3836744945627643,
        0.011511491472573064
      ],
      "p_permutation": 0.27556,
      "bootstrap_finite": 9542,
      "p_holm_24": 1.0
    }
  ],
  "sensitivities": {
    "alternative_score": {
      "core": {
        "n_tasks": 50,
        "rho": -0.05557411106680546,
        "ci95": [
          -0.3389038891285442,
          0.2273213091381526
        ],
        "p_permutation": 0.70266,
        "bootstrap_finite": 10000
      },
      "hit": {
        "n_tasks": 50,
        "rho": -0.09108413375060473,
        "ci95": [
          -0.37392126839720735,
          0.18603119579679175
        ],
        "p_permutation": 0.52725,
        "bootstrap_finite": 10000
      }
    },
    "input_score": {
      "core": {
        "n_tasks": 50,
        "rho": 0.2911902231222321,
        "ci95": [
          -0.06258347858755653,
          0.5924080483761314
        ],
        "p_permutation": 0.03993,
        "bootstrap_finite": 10000
      },
      "hit": {
        "n_tasks": 50,
        "rho": 0.24429278902982254,
        "ci95": [
          -0.11854970498261466,
          0.5640536157624145
        ],
        "p_permutation": 0.08261,
        "bootstrap_finite": 10000
      }
    },
    "exclude_two_examples": {
      "core": {
        "n_tasks": 48,
        "rho": 0.1514610530146301,
        "ci95": [
          -0.17354695673526402,
          0.45473871862405324
        ],
        "p_permutation": 0.30267,
        "bootstrap_finite": 10000
      },
      "hit": {
        "n_tasks": 48,
        "rho": 0.17567544157888001,
        "ci95": [
          -0.1417121872458575,
          0.46844837047382965
        ],
        "p_permutation": 0.2178,
        "bootstrap_finite": 10000
      }
    },
    "exclude_opus": {
      "core": {
        "n_tasks": 50,
        "rho": 0.14943007563553856,
        "ci95": [
          -0.17021940109328773,
          0.44135416325760723
        ],
        "p_permutation": 0.30135,
        "bootstrap_finite": 10000
      },
      "hit": {
        "n_tasks": 50,
        "rho": 0.0936176511851456,
        "ci95": [
          -0.22910235942513352,
          0.40286143265758345
        ],
        "p_permutation": 0.50541,
        "bootstrap_finite": 10000
      }
    },
    "partial_rank_lengths": {
      "length_unit": "Unicode characters",
      "reference": "finding plus key_points",
      "controls": [
        "question_length",
        "reference_length"
      ],
      "core": {
        "rho": 0.1428076767861947,
        "ci95": [
          -0.19931501352048417,
          0.45095025535618216
        ]
      },
      "hit": {
        "rho": 0.14946580775583745,
        "ci95": [
          -0.1821993785071193,
          0.4664815866974523
        ]
      }
    },
    "leave_one_out": {
      "core": {
        "rho_range": [
          0.11924814229440524,
          0.22270371885339507
        ]
      },
      "hit": {
        "rho_range": [
          0.1170634634908283,
          0.26212255066707196
        ]
      }
    }
  }
}