{
 "dataset": {
  "archive": "dandi",
  "id": "000620",
  "label": "DANDI:000620"
 },
 "page": "datasets/dandi/000620/",
 "generated": "2026-11-05T06:30:02+00:00",
 "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.",
 "outcome_counts": {
  "reproduced": 7,
  "partially_reproduced": 5,
  "not_reproduced": 0,
  "blocked": 14
 },
 "outcome_counts_with_outside_data": {
  "reproduced": 0,
  "partially_reproduced": 0,
  "not_reproduced": 0,
  "blocked": 0
 },
 "outcome_counts_elsewhere": {
  "reproduced": 0,
  "partially_reproduced": 0,
  "not_reproduced": 0,
  "blocked": 0
 },
 "outcome_counts_links": {
  "reproduced": 0,
  "partially_reproduced": 0,
  "not_reproduced": 0,
  "blocked": 0
 },
 "attempts": [
  {
   "id": "000620-watters-2026-01",
   "dataset": {
    "archive": "dandi",
    "id": "000620",
    "version": "0.261019.2000",
    "doi": "10.48324/dandi.000620/0.261019.2000"
   },
   "key": "dandi:000620",
   "label": "DANDI:000620",
   "dataset_version": "0.261019.2000",
   "paper": "10.64898/2026.01.27.702062",
   "paper_title": "Working Memory of Multi-Object Scenes in Primate Frontal Cortex",
   "date": "2026-11-04",
   "kind": "agent",
   "accountable": "CatalystNeuro",
   "agent": {
    "harness": "Claude Code",
    "model": "claude-opus-5-5",
    "protocol": "pilot-13"
   },
   "runner": {
    "platform": "GitHub Actions",
    "url": "https://github.com/catalystneuro/encore/actions/runs/41886879686"
   },
   "tier": "agent_verified",
   "deprecated": "",
   "deprecated_on": null,
   "entry": "000620-watters-2026",
   "entry_title": "Multi-object working memory in macaque frontal cortex (Watters et al. 2026)",
   "description": "Tests the findings of Watters, Gabel, Tenenbaum and Jazayeri (2026) on dandiset 000620, which holds the behaviour and spike-sorted units of 46 physiology sessions from two macaques in DMFC and FEF during a multi-object working-memory task (24 Triangle and 22 Ring sessions). It tests behaviour, unit counts, delay-phase selectivity, signal and noise correlations and the decoding analyses; the claims that rest on fitting the paper's Slot, Switching and Gain encoding models are blocked as too large to run here.",
   "notebooks": [
    {
     "path": "reproduce_watters_2026.ipynb",
     "title": "Behaviour, units, selectivity and correlations on dandiset 000620",
     "executed": "notebooks/000620-watters-2026/reproduce_watters_2026.html"
    },
    {
     "path": "decoding_watters_2026.ipynb",
     "title": "Decoding analyses (Extended Data Fig. 3) on dandiset 000620",
     "executed": "notebooks/000620-watters-2026/decoding_watters_2026.html"
    },
    {
     "path": "decoding_variants_watters_2026.ipynb",
     "title": "Variants of the decoding analyses (input scale, Ring output) on dandiset 000620",
     "executed": "notebooks/000620-watters-2026/decoding_variants_watters_2026.html"
    }
   ],
   "outcomes": [
    {
     "claim": "fig1b-performance-load",
     "statement": "In both monkeys accuracy falls as the number of objects grows, yet stays far above the chance of saccading to a random stimulus object for two- and three-object stimuli.",
     "source": "Results, Task, Behavior, and Hypotheses, paragraph 1; Figure 1b",
     "quote": "Animals learned the task and exhibited variable reaction times, with error rates increasing as the number of objects grew ... However, performance remained far above chance for two- and three-object stimuli",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Under both readings and both radii the fraction correct falls from 1 to 2 to 3 objects in both monkeys and the 2- and 3-object fractions are above 1/2 and 1/3. Reading (a), Triangle trials on the triangle, code radius, gives P 0.953, 0.930, 0.692 and E 0.928, 0.759, 0.561, equal to the values the paper's notebook printed; with the Methods radius P 0.901, 0.887, 0.661 and E 0.903, 0.749, 0.555. Reading (b), Triangle and Ring pooled, code radius, P 0.958, 0.911, 0.692 and E 0.939, 0.757, 0.561. The smallest margin above chance is 0.222 (E, 3 objects, Methods radius). The paper gives no test.",
     "deviations": [
      "Physiology sessions only (24 Triangle, 22 Ring); the behaviour-only session Perle 2022-06-08 and the 2-object-only session Perle 2022-06-06 are left out, as the Fig. 1b legend says all physiology sessions."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig1c-reaction-times",
     "statement": "Reaction times, from cue onset to the response saccade, vary from trial to trial, with trials faster than 180 ms counted as broken fixation.",
     "source": "Results, Task, Behavior, and Hypotheses, paragraph 1; Figure 1c legend",
     "quote": "Animals learned the task and exhibited variable reaction times",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Reaction times of trials at or above 180 ms have interquartile ranges of 52.0 ms (P) and 69.8 ms (E) under reading (a) and 50.6 and 73.1 ms under reading (b), with medians of 230.0, 268.5, 227.4 and 273.4 ms. Before the cut, 1.7% (P) and 0.4% (E) of reading (a) trials are under 180 ms. Incorrect trials are faster than correct ones in both monkeys (medians P 220.7 against 232.2 ms, E 260.7 against 272.4 ms, reading (a)). The cue phase lasts 250 ms in the data, not 180 ms, and 47.9% of completed trials have a reaction time under 250 ms.",
     "deviations": [
      "Reaction time is response_time minus cue onset, as the code computes it; response_time reads as saccade onset (the closed-loop response time is 134 ms later at the median), which the paper's legend calls onset of the response saccade."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "ext1-misbinding-errors",
     "statement": "A common error is a saccade to the location of a non-target stimulus object, reporting the cued identity at the wrong location.",
     "source": "Discussion, Position-identity binding, paragraph 1; Extended Data Fig. 1",
     "quote": "A common error in our task occurs when the monkey reports the identity of an object at the wrong location ( Extended Data Fig. 1 ), consistent with misbinding in working memory",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "With the code's radius (0.2 arena widths), errors at a non-target object's vertex are 92.3% (E, 2 objects), 99.4% (E, 3), 84.5% (P, 2) and 96.5% (P, 3) of errors, the largest class in each, so both readings hold. With the Methods radius (2.4 degrees, 0.1 arena widths) both readings hold for E at 2 and 3 objects (86.5%, 96.7%) and P at 3 (82.0%), and fail for P at 2 objects, where 207 errors are at the non-target vertex and 249 at no vertex (44.9%). The outcome depends on the radius; the code's is adopted (see deviations). The question is put to the paper's authors (findings).",
     "deviations": [
      "Assumed the correctness radius of 0.2 arena widths (4.8 degrees), the code's (load_triangle_data.py, correct_threshold 0.2, also used for response_location), where the Methods (Feedback phase) say 2.4 degrees. Reason: in the data every completed trial with a closed-loop error under 0.2 arena widths was rewarded (49,185 trials) and none at 0.2 or more (12,469 trials), so 0.2 is the radius the task used to score responses, and the code that made Extended Data Fig. 1 assigns responses to vertices with it. The Methods radius is shown and gives a different outcome for Monkey P with 2 objects.",
      "Errors at the target vertex but outside the radius cannot occur here, since the vertex radius equals the correctness radius; responses at no vertex and at an empty vertex are counted apart."
     ],
     "assumed": true,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "triangle-unit-counts",
     "statement": "The Triangle dataset holds 4,688 neurons, 2,937 in DMFC and 1,751 in FEF, from 24 sessions, 10 from Monkey P and 14 from Monkey E.",
     "source": "Results, The Triangle Dataset, paragraph 1; Methods, Neurophysiology, last paragraph",
     "quote": "We recorded 4,688 well-isolated neurons (2,937 from DMFC and 1,751 from FEF) in the Triangle dataset across 24 sessions",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Reading (a), good and multi-units with delay-phase mean rate above 1 Hz over their completed stable trials, gives 4,688 units (2,937 DMFC, 1,751 FEF) from 10 Perle and 14 Elgar sessions, exactly the paper's numbers. Reading (b), also stable on at least 500 trials, gives 4,473 (2,857, 1,616), within tolerance. Reading (c), good units only, gives 1,563 (1,110, 453) and does not match. Of all 4,899 sorted Triangle units, 4,688 pass reading (a).",
     "deviations": [
      "DMFC is probe s0 and FEF the V-probes, from the claims file and the code; the dandiset's electrodes tables give no location."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "ring-unit-counts",
     "statement": "The Ring dataset holds 3,051 neurons, 1,712 in DMFC and 1,339 in FEF, from 22 sessions, 10 from Monkey P and 12 from Monkey E.",
     "source": "Results, Replication of the Results with the Ring Dataset, paragraph 1; Methods, Neurophysiology, last paragraph",
     "quote": "We recorded 3,051 well-isolated neurons (1,712 from DMFC and 1,339 from FEF) across 22 neurophysiology sessions",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Reading (a) gives 3,052 units (1,712 DMFC, 1,340 FEF) from 10 Perle and 12 Elgar Ring sessions, against 3,051, 1,712 and 1,339. Reading (b) gives 2,936 (1,655, 1,281), within tolerance; reading (c) 1,123 and does not match. The Ring sessions are Perle 2022-06-07 and 06-09 to 06-19 (10) and Elgar 2022-09-06 to 09-20 (12); the code's model comparison lists 20, without Elgar 2022-09-18 and 09-19 (2,862 units under reading (a)). Adding Perle 2022-06-06 would give 3,121 units from 11 Perle sessions.",
     "deviations": [
      "Perle 2022-06-06, which has spike sorting but only 2-object trials, is not counted as a Ring session; the Ring design has 1- and 2-object trials (Methods, Behavioral Task)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "ext2-stimulus-delay-activity",
     "statement": "Neurons are typically active in both the stimulus and the delay phase, and a unit's spatial preference is similar across the two phases.",
     "source": "Results, The Triangle Dataset, paragraph 1; Extended Data Fig. 2a and 2b",
     "quote": "We found that neurons were typically active in both stimulus and delay phases ( Extended Data Fig. 2 )",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Panel a, Triangle units above 1 Hz, stimulus against delay mean rate: Pearson r 0.898 (P, 1,536 units) and 0.971 (E, 3,152), Spearman 0.894 and 0.963, p below 1e-300; 97.5% (P) and 99.1% (E) of units are above 1 Hz in the stimulus phase. Panel b, normalised 1-object rates per unit and vertex, stimulus against delay: Pearson 0.600 (P DMFC), 0.488 (P FEF), 0.398 (E DMFC), 0.440 (E FEF), Spearman 0.371 to 0.545, all p below 1e-89; all four plots pass both readings.",
     "deviations": [
      "The stimulus phase is stimulus onset to delay onset, as the code computes it; it lasts 1.000 s at the median (0.9997 to 1.4167 s)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig2b-selectivity-range",
     "statement": "Delay-phase position and identity AUCs on 1-object trials span a wide range across units.",
     "source": "Results, The Triangle Dataset, paragraph 1; Figure 2b",
     "quote": "A Receiver Operating Characteristic analysis of firing rates in the delay phase revealed a wide range of selectivities for position and identity across neurons ( Fig. 2b )",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "partially_reproduced",
     "blocked_by": null,
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Position AUCs meet the rule in both monkeys under both p-value methods (lowest decile 0.538 to 0.574, highest decile 0.706 to 0.897). Identity AUCs do not: their highest decile is 0.654 (E) and 0.700 (P) with the Methods p-value, 0.681 (E) and 0.715 (P) with the code's, so only P under the code's p passes. Significant and non-significant units are present everywhere (Methods p, P 955 and 581, E 1,156 and 1,996). Identity AUCs reach 0.969 (P) and 1.000 (E). With the 500-trial rule no conclusion changes. The deciles are the claims file's reading of a wide range. The criterion defines only the reproduced case; partially reproduced is this attempt's label for a result that holds in part.",
     "deviations": [
      "AUCs are not cross-validated; the Fig. 2b legend calls them cross-validated but neither the Methods nor the code says how (claims file, T6).",
      "AUC by ranks with ties as half; the code's trapezoid on value-sorted labels orders ties arbitrarily.",
      "The code's p-value (auc.py) resamples labels with replacement in the value-sorted order, a null of 5,000 draws, where the Methods shuffle 1,000 times; both are run. Counts of units entering the models under the code's p-value run above the counts its selectivity notebook printed for Elgar, for reasons not found."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig5ab-ring-behavior",
     "statement": "Both monkeys perform well on the Ring dataset, with reaction-time variability like that on the Triangle dataset.",
     "source": "Results, Replication of the Results with the Ring Dataset, paragraph 1; Figure 5a and 5b; Extended Data Fig. 9a and 9b (the Results cite \"Extended Data Fig. 8a-b\", but the per-monkey Ring figure is the ninth Extended Data legend)",
     "quote": "Both monkeys performed well on the Ring dataset and exhibited reaction-time variability similar to the Triangle dataset",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Completed Ring trials, all conditions: 2-object fraction correct P 0.903 and E 0.756 (code radius), P 0.786 and E 0.721 (Methods radius), above 1/2; 1-object P 0.960 and E 0.945 (code radius). Without the left/right trials (1,525 of 29,299, 5.2%) P 0.899 and E 0.749. Ring against Triangle interquartile range of reaction time, ratio 0.935 (P, 48.6 against 52.0 ms) and 1.098 (E, 76.6 against 69.8 ms).",
     "deviations": [
      "The Triangle reference for reaction-time variability is reading (a) of fig1c-reaction-times (Triangle trials on the triangle, trials at or above 180 ms)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig5cd-ring-tuning",
     "statement": "Ring neurons in DMFC and FEF show a variety of spatial receptive fields and diverse delay-phase position and identity selectivities, as in the Triangle dataset.",
     "source": "Results, Replication of the Results with the Ring Dataset, paragraph 1; Figure 5c and 5d; Extended Data Fig. 9c",
     "quote": "These neurons showed a variety of spatial receptive fields in both DMFC and FEF ( Fig. 5c ; Extended Data Fig. 9c ) and diverse selectivities to position and identity in delay-phase firing rates ( Fig. 5d ), similar to the Triangle dataset",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "partially_reproduced",
     "blocked_by": null,
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Variety of receptive fields holds: units with significant position AUC have preferred angles in all three sectors in each area of each monkey (Methods p, fewest 48 in a cell; code p, fewest 72). The AUC part judged as fig2b fails: the highest decile of identity AUC is 0.611 (E), 0.630 (P) and 0.623 pooled with the Methods p (0.638, 0.653, 0.646 with the code's), and Elgar's position AUCs reach a highest decile of only 0.658 (0.674); Perle's position AUCs (0.796) and the pooled ones (0.740) pass. The criterion defines only the reproduced case; partially reproduced is this attempt's label for a result that holds in part.",
     "deviations": [
      "The preferred angle is the peak of the mean of the 100 bootstrapped tuning curves; the Methods describe the curves, not how a preference is read from them.",
      "AUCs are not cross-validated, as for fig2b-selectivity-range."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig2d-signal-noise-triangle",
     "statement": "In the Triangle dataset, pairs of neurons with similar 1-object tuning have positive noise correlations on 2-object trials, and dissimilarly tuned pairs negative ones.",
     "source": "Results, The Triangle Dataset, paragraph 2; Figure 2d",
     "quote": "Pairs of neurons with similar selectivities on 1-object trials tended to have positive noise correlations on 2-object trials ... Conversely, dissimilarly tuned pairs of neurons tended to have negative noise correlations.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "methods_underspecified",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "The criterion's two methods give different outcomes, which it settles as methods_underspecified. Main method (Methods, Noise Correlations), 55,766 P and 328,273 E pairs: lowest bin P -0.0041 (95% interval -0.0071 to -0.0011), E -0.0230 (-0.0244 to -0.0216); highest bin P 0.0574 (0.0545 to 0.0602), E 0.0490 (0.0475 to 0.0504), so it reproduces the claim in both monkeys. Code variant (run_compute_triangle.py): P lowest -0.0189 (-0.0292 to -0.0085), highest 0.0769; E lowest bin (124 rows) 0.0245 (-0.0296 to 0.0786), highest 0.1412 (0.0975 to 0.1849), so only the rise holds for E (partially reproduced). Per session, signal and noise correlation correlate positively in all 24 sessions (Pearson 0.100 to 0.442). The question of which signal correlation the figure uses is put to the paper's authors (findings). Counting each pair once, with intervals clustered by pair (up to 18 rows per pair in the code variant), changes nothing: E's lowest bin, 124 rows from 98 pairs, is 0.0245 (-0.0327 to 0.0818), its highest 0.1412 (0.0948 to 0.1877), and P's lowest stays below zero (-0.0309 to -0.0068).",
     "deviations": [
      "Code variant vectorised: the within-condition shuffle permutes trials for all units at once and each pair's shared trials are taken after it, where the code keeps the pair's shared trials first and then shuffles.",
      "Intervals are normal 95% intervals of the mean, a standard choice; the paper's plots use seaborn's bootstrap interval."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig5e-signal-noise-ring",
     "statement": "In the Ring dataset, noise correlation rises with signal correlation, as in the Triangle dataset.",
     "source": "Results, Replication of the Results with the Ring Dataset, paragraph 1; Figure 5e; Extended Data Fig. 9d",
     "quote": "We also found a positive relationship between signal correlation and noise correlation ( Fig. 5e ; Extended Data Fig. 9d ), corroborating our findings in the Triangle dataset",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "partially_reproduced",
     "blocked_by": null,
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Ordered pairs of Ring units above 1 Hz with at least 50 shared 1-object and 200 2-object trials, 294,992 P and 154,174 E. Both readings agree. P (s = 0.8): highest bin 0.1728 (0.1098 to 0.2359) against lowest -0.1083 (-0.1424 to -0.0742) under reading (a), 0.1620 against -0.0629 under (b); pooled the same. E (s = 0.3): highest 0.0238 (0.0151 to 0.0324) against lowest 0.0174 (0.0077 to 0.0271) under (a), 0.0229 against 0.0192 under (b), overlapping, so the criterion fails for E. The E means rise over the middle bins (0.0145 to 0.0214, reading (a)), and the across-pair Pearson correlation of signal and noise is 0.017 (E) and 0.061 (P). Each unordered pair appears twice (ordered pairs, as run_compute_ring.py); with intervals clustered by pair E's highest bin (0.0131 to 0.0344) still overlaps its lowest (0.0059 to 0.0288), and P's bins stay apart (0.0873 to 0.2584 against -0.1504 to -0.0663, reading (a)). The criterion defines only the reproduced case; partially reproduced is this attempt's label for a result that holds in part.",
     "deviations": [
      "Units above 1 Hz (T5); the code's run_compute_ring.py takes every unit of its selectivity table.",
      "Neighbours within each pair's shared trials are taken from one distance sort per session; ties among equidistant trials are broken by trial order, which can differ from the code's argsort.",
      "Methods reading (a) uses the 10 nearest neighbours without the trial itself, as the code does for reading (b)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "ext3ab-decoding",
     "statement": "Single-trial delay-phase activity carries information about object positions and identities, decoded above a shuffled-label control.",
     "source": "Results, The Triangle Dataset, paragraph 1; Extended Data Fig. 3a and 3b",
     "quote": "We also found that neural activity contained information about object positions and identities ( Extended Data Fig. 3a-b )",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "partially_reproduced",
     "blocked_by": null,
     "notebook": "decoding_watters_2026.ipynb",
     "notes": "Sessions Perle 2022-06-01, 2022-05-31 and 2022-06-04, the three with the most units entering the models, as in the code. Panel a, delay phase: position 99.2%, 95.7% and 82.5% against shuffled 33.8%, 33.3% and 32.1%, above control in all ten splits and by more than two standard deviations in each session; identity 50.0%, 46.2% and 30.0% against 37.5%, 32.4% and 30.8%, above control under neither reading in any session (0 of 3 under each). Identity decoders fit their training trials little better than decoders trained on shuffled labels (delay, 82.7% against 84.1%, 92.5% against 89.5%, 63.6% against 66.3%). The stimulus phase gives the same pattern. Panel b: the absent location 78.5% to 99.7%, above the legend's 66% in every session and phase; the absent identity 32.4% to 43.6%, below it. Position holds and identity does not; the criterion defines only the reproduced case, and partially reproduced is this attempt's label for that. With inputs z-scored per unit (variants notebook, not the code's) identity is above control in one session under each reading (delay 50.8%, 48.1%, 32.9% against 35.0%, 31.4%, 30.8%; all ten splits only in 2022-05-31, two standard deviations only in 2022-06-01), so it still fails. Whether the criterion's identity test matches the paper, whose Methods say identity was poorly decodable on single trials, is asked of a reader (questions/000620.yaml).",
     "deviations": [
      "The decoders are models.py's LinearModel1Object written in numpy (wm.train_linear) with the configs' settings (panel a SGD lr 0.01, panel b Adam lr 0.001, L1 0.003, batch 512, 10,000 steps), batched over splits; torch's random streams are not reproduced.",
      "Inputs are rates in Hz with no per-unit normalisation, as the code's dataset.py, where the normalisation lines are commented out (\"# Normalize neural data\").",
      "Units entering the models are selected with the Methods' AUC p-value; this picks the same sessions as the code's scripts name. Ten seeds as the legend says, where run_1_triangle.py lists twenty.",
      "The code's filters (dataset.py, units with 50 trials, trials with 5 units, then greedy removal to a fully observed matrix, which with keep_units_weight 1.0 drops units before trials) leave 105 to 121 1-object trials, and 104, 82 and 20 units; for 2022-06-04, 72 of 116 units have 50 trials and 20 are kept."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "ext3c-decoder-orthogonality",
     "statement": "The coding axes of a two-object decoder come out nearly orthogonal even for synthetic data with no slot structure, so decoder orthogonality does not diagnose slots.",
     "source": "Methods, Decoding Analyses, paragraphs 1 and 4; Extended Data Fig. 3c",
     "quote": "However, we found that this orthogonality resulted even on synthetic data that was specifically not Slot-like. In other words, orthogonality of these decoding subspaces is not diagnostic of object slots.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "partially_reproduced",
     "blocked_by": null,
     "notebook": "decoding_watters_2026.ipynb",
     "notes": "Median angles between corresponding columns of the two decoders, over sessions, ten seeds and columns, with the code's decoder (rates in Hz, sigmoid outputs). Receptive Field data 91.9 and 91.8 degrees (Triangle delay, stimulus) and 96.4 and 96.7 (Ring), within 10 degrees of the Orthogonal data (91.0, 90.8, 92.5, 95.0). Ring neural data 88.8 and 93.2, so both readings hold for the Ring. Triangle neural data 106.7 (delay) and 106.6 (stimulus), outside 75 to 105 and 15.7 and 15.8 degrees from the Orthogonal data, so neither reading holds there; per session 96.8 to 114.6, with all three delay-phase sessions above 105 (105.3 to 110.7), and without 2022-06-04 (20 units) 109.1 (delay) and 99.3 (stimulus). The claim holds for the Ring and fails for the Triangle neural data, so it is partially reproduced; the criterion has no rule for a result that holds in one dataset only, and partially reproduced is this attempt's label. The part of the statement about data without slots holds: the Receptive Field decoders come out near orthogonal in both datasets. The Ring pass rests on the code's sigmoid output (assumed, see deviations): with the linear output the Methods describe, the Ring neural medians are 140.2 and 142.1 and fail both readings, so the neural data would fail in both datasets, which by the criterion is not reproduced (the Triangle decoders were not run with a linear output). With z-scored inputs (variants notebook, not the code's) the Triangle neural data give 96.1 and 98.5 and pass both readings, while the Ring data fail reading (b) in 3 of 4 cases (medians 101.6 to 102.2 against 90.0; the Ring neural delay phase, 99.1, passes). So the outcome depends on the decoder's details.",
     "deviations": [
      "LinearModel2Objects written in numpy (wm.train_linear_2obj), Adam lr 0.001, L1 0.003, batch 512, 10,000 steps (config_2_triangle.py, config_2_ring.py); synthetic datasets as dataset.py with numpy's generator.",
      "Assumed: as in models.py the Ring decoders keep the sigmoid output, though the Ring targets 2x - 1 and 2y - 1 (labelers.py) lie in -0.7 to 0.7, while the Methods (2-object decoder) say \"The variables W 0 , B 0 parameterize one linear decoder\" and give no output function. The code's decoder is taken because it is the method that produced the paper's figure; the outcome rests on this, since with a linear output the Ring neural data fail (notes). Inputs are rates in Hz, as dataset.py.",
      "Ring sessions Perle 2022-06-10, 06-11 and 06-09, ranked by units entering the models with the Methods p-value, the sessions run_2_ring.py names. The code's filters leave 104, 85 and 20 Triangle units and 70, 90 and 44 Ring units, below the about 100 simultaneously recorded neurons the Methods describe in 5 of 6 sessions."
     ],
     "assumed": true,
     "disagreed": true,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig4a-gain-best-triangle",
     "statement": "On the Triangle dataset the Gain model has a higher cross-validated model likelihood than the Slot and Switching models, in 88% of sessions and in every session with at least 100 simultaneously recorded units.",
     "source": "Results, Modeling Results, paragraph 1; Figure 4a",
     "quote": "Across sessions, the Gain model had a higher cross-validated likelihood than the Slot and Switching models ( Fig. 4a ). This held for 88% of sessions, including all sessions with at least 100 simultaneously recorded units.",
     "statistics": [
      {
       "about": "Gain against Slot and against Switching, mean model likelihood, per monkey",
       "test": "significance of the difference of the mean (Figure 4a brackets)",
       "p": 0.05,
       "p_relation": "<",
       "n": "10 sessions of Monkey P and 14 of Monkey E, 10 train/test splits each"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the paper's encoding models fitted (T7). One gradient step at the paper's sizes (MLPs 512-1024-512, batch 4,096 samples of up to 3 objects, 59 modeled units, the median of a Triangle session) took 0.436 s in the latest run (the timing varies with the machine and its load; earlier runs gave 0.45, 0.84 and 0.98 s), so one fit of 10,000 steps takes about 1.2 h; this claim needs the kappa sweep (11 values, two splits, 10,000 steps) and ten splits for 3 models on 24 sessions, 23,040,000 steps or about 116 days, far over the 60-minute notebook limit and the run's 5 hours. The estimate is a lower bound (no Slot assignment losses, evaluation or data pipeline); the machine has no GPU, and at 100 times the speed the model-fitting estimates would still be 1.5 to 27.9 h, each over the 60-minute notebook limit. No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "ext6-gain-best-areas",
     "statement": "The Gain model's advantage holds in DMFC alone, in FEF alone, and in single neurons alone.",
     "source": "Results, Modeling Results, paragraph 1; Extended Data Fig. 6",
     "quote": "The results were consistent across both DMFC and FEF ( Extended Data Fig. 6 )",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the paper's encoding models fitted (T7). One gradient step at the paper's sizes (MLPs 512-1024-512, batch 4,096 samples of up to 3 objects, 59 modeled units, the median of a Triangle session) took 0.436 s in the latest run (the timing varies with the machine and its load; earlier runs gave 0.45, 0.84 and 0.98 s), so one fit of 10,000 steps takes about 1.2 h; this claim needs 3 restrictions x 3 models on 8 sessions, 23,040,000 steps or about 116 days, far over the 60-minute notebook limit and the run's 5 hours. The estimate is a lower bound (no Slot assignment losses, evaluation or data pipeline); the machine has no GPU, and at 100 times the speed the model-fitting estimates would still be 1.5 to 27.9 h, each over the 60-minute notebook limit. No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "ext5-general-slot",
     "statement": "Letting slots be arbitrary linear subspaces (the General Slot model) does not change the model comparison; Gain still wins.",
     "source": "Results, Slot hypothesis, paragraph 1; Methods, General Slot Model; Extended Data Fig. 5",
     "quote": "we found that relaxing this assumption and allowing the slots to be linear subspaces in neural activity space does not qualitatively change our results ( Extended Data Fig. 5 )",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the paper's encoding models fitted (T7). One gradient step at the paper's sizes (MLPs 512-1024-512, batch 4,096 samples of up to 3 objects, 59 modeled units, the median of a Triangle session) took 0.436 s in the latest run (the timing varies with the machine and its load; earlier runs gave 0.45, 0.84 and 0.98 s), so one fit of 10,000 steps takes about 1.2 h; this claim needs the General Slot model on 24 sessions, 7,680,000 steps or about 39 days, far over the 60-minute notebook limit and the run's 5 hours. The estimate is a lower bound (no Slot assignment losses, evaluation or data pipeline); the machine has no GPU, and at 100 times the speed the model-fitting estimates would still be 1.5 to 27.9 h, each over the 60-minute notebook limit. No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig4b-identifiability",
     "statement": "Applied to synthetic spike trains sampled from each fitted model, the full fitting and comparison pipeline recovers the generating model, Slot, Switching or Gain.",
     "source": "Results, Modeling Results, paragraph 3; Methods, Validation Using Synthetic Data; Figure 4b",
     "quote": "This analysis confirmed that our model comparison reliably recovered the ground truth for each of the Slot, Switching, and Gain models ( Fig. 4b )",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the paper's encoding models fitted (T7). One gradient step at the paper's sizes (MLPs 512-1024-512, batch 4,096 samples of up to 3 objects, 59 modeled units, the median of a Triangle session) took 0.436 s in the latest run (the timing varies with the machine and its load; earlier runs gave 0.45, 0.84 and 0.98 s), so one fit of 10,000 steps takes about 1.2 h; this claim needs refits on synthetic data from 10 sessions, 21,600,000 steps or about 109 days, after the fits of fig4a that generate the data, far over the 60-minute notebook limit and the run's 5 hours. The estimate is a lower bound (no Slot assignment losses, evaluation or data pipeline); the machine has no GPU, and at 100 times the speed the model-fitting estimates would still be 1.5 to 27.9 h, each over the 60-minute notebook limit. No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "nslots-two-best",
     "statement": "Two slots give the best cross-validated likelihood for the Slot model on both datasets and both monkeys.",
     "source": "Methods, Slot Model, paragraphs 1 and 2",
     "quote": "Through hyperparameter sweeping, we found that N slots = 2 provided the best cross-validated performance on both the Triangle and Ring dataset for both monkeys.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the paper's encoding models fitted (T7). One gradient step at the paper's sizes (MLPs 512-1024-512, batch 4,096 samples of up to 3 objects, 59 modeled units, the median of a Triangle session) took 0.436 s in the latest run (the timing varies with the machine and its load; earlier runs gave 0.45, 0.84 and 0.98 s), so one fit of 10,000 steps takes about 1.2 h; this claim needs 4 slot counts x 2 splits on 4 sessions per monkey and dataset, 1,280,000 steps or about 6 days, far over the 60-minute notebook limit and the run's 5 hours. At 100 times the speed this sweep (1.5 h) would fit in the run's 5 hours but not in the notebook limit. The estimate is a lower bound (no Slot assignment losses, evaluation or data pipeline); the machine has no GPU, and at 100 times the speed the model-fitting estimates would still be 1.5 to 27.9 h, each over the 60-minute notebook limit. No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "ext11-kappa-tuning",
     "statement": "For most sessions the best temporal smoothness kappa lies inside the sweep, is broadly the same across models, and is lower in sessions with more units.",
     "source": "Methods, Tuning the Temporal Smoothness and Training Steps, paragraphs 1 and 2; Extended Data Fig. 11",
     "quote": "We found that for most sessions, cross-validated likelihood was maximized for a value of \u03ba somewhere in the middle of our sweep. Furthermore, this optimal value of \u03ba was broadly consistent across models.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the paper's encoding models fitted (T7). One gradient step at the paper's sizes (MLPs 512-1024-512, batch 4,096 samples of up to 3 objects, 59 modeled units, the median of a Triangle session) took 0.436 s in the latest run (the timing varies with the machine and its load; earlier runs gave 0.45, 0.84 and 0.98 s), so one fit of 10,000 steps takes about 1.2 h; this claim needs the kappa sweeps of fig4a, part of its 23,040,000 steps (about 116 days), far over the 60-minute notebook limit and the run's 5 hours. The estimate is a lower bound (no Slot assignment losses, evaluation or data pipeline); the machine has no GPU, and at 100 times the speed the model-fitting estimates would still be 1.5 to 27.9 h, each over the 60-minute notebook limit. No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "ext7-gains-distributed",
     "statement": "The Gain model's per-trial gains are spread across objects and do not pick one object per trial.",
     "source": "Results, Gain model predictions for behavioral errors and reaction times, paragraph 1; Extended Data Fig. 7c",
     "quote": "These gains were distributed across the objects and did not sparsely select one object per trial ( Extended Data Fig. 7 )",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the gains or attention of fitted models, from fig4a-gain-best-triangle, which is blocked as too large (one fit of 10,000 steps takes about 1.2 h here; the Triangle comparison about 116 days, the Ring one about 107 days). No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig4c-gain-performance-triangle",
     "statement": "On 3-object Triangle trials, performance is higher when the target object's neurally inferred delay gain is higher (R = 0.35 for Monkey P, 0.26 for Monkey E).",
     "source": "Figure 4c legend; Results, Gain model predictions for behavioral errors and reaction times, paragraph 1",
     "quote": "The regression is significantly positive for both monkeys ( R = 0. 35\u00b10. 04, p < 10 \u22129 for Monkey P; R = 0. 26\u00b10. 04, p < 10 \u22127 for Monkey E) ... Strikingly, performance was higher for objects with higher delay-phase gains ( Fig. 4c )",
     "statistics": [
      {
       "about": "Monkey P",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = 0.35 \u00b1 0.04",
       "p": 1e-09,
       "p_relation": "<"
      },
      {
       "about": "Monkey E",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = 0.26 \u00b1 0.04",
       "p": 1e-07,
       "p_relation": "<"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the gains or attention of fitted models, from fig4a-gain-best-triangle, which is blocked as too large (one fit of 10,000 steps takes about 1.2 h here; the Triangle comparison about 116 days, the Ring one about 107 days). No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig4d-gain-rt-triangle",
     "statement": "On correct 3-object Triangle trials, the residual target gain is negatively correlated with residual reaction time (R = -0.14 for Monkey P, -0.04 for Monkey E).",
     "source": "Figure 4d legend; Results, Gain model predictions for behavioral errors and reaction times, paragraph 1",
     "quote": "The regression slope is significantly negative for both monkeys ( R =\u2212 0. 14\u00b10. 016, p < 10 \u221210 for Monkey P; R =\u2212 0. 04\u00b10. 015, p = 0. 007 for Monkey E) ... residual gains were negatively correlated with residual reaction times ( Fig. 4d )",
     "statistics": [
      {
       "about": "Monkey P",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = -0.14 \u00b1 0.016",
       "p": 1e-10,
       "p_relation": "<"
      },
      {
       "about": "Monkey E",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = -0.04 \u00b1 0.015",
       "p": 0.007
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the gains or attention of fitted models, from fig4a-gain-best-triangle, which is blocked as too large (one fit of 10,000 steps takes about 1.2 h here; the Triangle comparison about 116 days, the Ring one about 107 days). No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "ext8-slot-switching-behavior",
     "statement": "The relations of attention to accuracy and reaction time are weaker for the Slot and Switching models than for the Gain model.",
     "source": "Results, Gain model predictions for behavioral errors and reaction times, paragraph 1; Extended Data Fig. 8 legend",
     "quote": "These behavioral relationships were weaker for the Slot and Switching models ( Extended Data Fig. 8 )",
     "statistics": [
      {
       "about": "Slot attention against reaction time, Monkey P (panel b)",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = -0.03 \u00b1 0.017",
       "p": 0.034
      },
      {
       "about": "Slot attention against reaction time, Monkey E (panel b)",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = -0.01 \u00b1 0.014",
       "p": 0.4
      },
      {
       "about": "Switching attention against performance, Monkey P (panel c)",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = 0.65 \u00b1 0.02",
       "p": 1e-10,
       "p_relation": "<"
      },
      {
       "about": "Switching attention against performance, Monkey E (panel c)",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = 0.34 \u00b1 0.04",
       "p": 1e-10,
       "p_relation": "<"
      },
      {
       "about": "Switching attention against reaction time, Monkey P (panel d)",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = -0.05 \u00b1 0.016",
       "p": 0.002
      },
      {
       "about": "Switching attention against reaction time, Monkey E (panel d)",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = -0.03 \u00b1 0.014",
       "p": 0.063
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the gains or attention of fitted models, from fig4c-gain-performance-triangle and fig4d-gain-rt-triangle, which is blocked as too large (one fit of 10,000 steps takes about 1.2 h here; the Triangle comparison about 116 days, the Ring one about 107 days). No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig5f-gain-best-ring",
     "statement": "On the Ring dataset the Gain model again has higher model likelihood than the Slot and Switching models, for both monkeys.",
     "source": "Results, Replication of the Results with the Ring Dataset, paragraph 2; Figure 5f",
     "quote": "As in the Triangle dataset, the Gain model had higher model likelihood than the Slot and Switching models for both monkeys ( Fig. 5f )",
     "statistics": [
      {
       "about": "Gain against Slot and against Switching, per monkey",
       "test": "significance of the difference of the mean (Figure 4a format)",
       "p": 0.05,
       "p_relation": "<"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the paper's encoding models fitted (T7). One gradient step at the paper's sizes (MLPs 512-1024-512, batch 4,096 samples of up to 3 objects, 59 modeled units, the median of a Triangle session) took 0.436 s in the latest run (the timing varies with the machine and its load; earlier runs gave 0.45, 0.84 and 0.98 s), so one fit of 10,000 steps takes about 1.2 h; this claim needs 3 models on 22 Ring sessions, 21,120,000 steps or about 107 days, far over the 60-minute notebook limit and the run's 5 hours. The estimate is a lower bound (no Slot assignment losses, evaluation or data pipeline); the machine has no GPU, and at 100 times the speed the model-fitting estimates would still be 1.5 to 27.9 h, each over the 60-minute notebook limit. No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig5g-gain-performance-ring",
     "statement": "On 2-object Ring trials, performance rises with the target object's gain (R = 0.042 for Monkey P, 0.01 for Monkey E).",
     "source": "Results, Replication of the Results with the Ring Dataset, paragraph 2; Figure 5g legend",
     "quote": "Regression is significantly positive ( R = 0. 042\u00b10. 003, p < 10 \u221210 ) ... Regression is significantly positive ( R = 0. 01\u00b10. 003, p = 0. 006)",
     "statistics": [
      {
       "about": "Monkey P",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = 0.042 \u00b1 0.003",
       "p": 1e-10,
       "p_relation": "<"
      },
      {
       "about": "Monkey E",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = 0.01 \u00b1 0.003",
       "p": 0.006
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the gains or attention of fitted models, from fig5f-gain-best-ring, which is blocked as too large (one fit of 10,000 steps takes about 1.2 h here; the Triangle comparison about 116 days, the Ring one about 107 days). No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig5h-gain-rt-ring",
     "statement": "On correct 2-object Ring trials, residual target gain is negatively correlated with residual reaction time (R = -0.13 for Monkey P, -0.03 for Monkey E).",
     "source": "Results, Replication of the Results with the Ring Dataset, paragraph 2; Figure 5h legend; Methods, Ring Dataset Normalized Target Gain",
     "quote": "The regression is significantly negative for both monkeys ( R =\u2212 0. 13\u00b10. 011, p < 10 \u221210 for Monkey P, R =\u2212 0. 03\u00b10. 012, p = 0. 018 for Monkey E)",
     "statistics": [
      {
       "about": "Monkey P",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = -0.13 \u00b1 0.011",
       "p": 1e-10,
       "p_relation": "<"
      },
      {
       "about": "Monkey E",
       "test": "linear regression (Pearson correlation)",
       "statistic": "R = -0.03 \u00b1 0.012",
       "p": 0.018
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "too_large",
     "notebook": "reproduce_watters_2026.ipynb",
     "notes": "Needs the gains or attention of fitted models, from fig5f-gain-best-ring, which is blocked as too large (one fit of 10,000 steps takes about 1.2 h here; the Triangle comparison about 116 days, the Ring one about 107 days). No fit was started.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    }
   ],
   "findings": [
    {
     "kind": "code_mismatch",
     "statement": "The Methods (Feedback phase) say a response was correct within 2.4 degrees (0.1 arena widths) of the target; the paper's code scores correctness at 0.2 arena widths (4.8 degrees), and the data agree with the code: every completed trial with a closed-loop response error under 0.2 arena widths was rewarded (49,185) and none at 0.2 or more (12,469).",
     "found": null,
     "status": null
    },
    {
     "kind": "paper_inconsistency",
     "statement": "The Methods say the cue phase lasted 180 ms and the Fig. 1c legend counts saccades earlier than 180 ms after cue onset as broken fixation; in the trials tables the response phase begins 250 ms after cue onset (0.2498 to 0.3167 s), 47.9% of completed trials have a response time under 250 ms, and 1.1% are under 180 ms yet are not marked as broken fixation.",
     "found": null,
     "status": null
    },
    {
     "kind": "metadata_insufficient",
     "statement": "The trials column response_time is described as the response time from high-resolution eye tracking, without saying which event it marks; it precedes closed_loop_response_time by 134 ms at the median and in some completed trials precedes cue onset (earliest 12.5 ms before), so it reads as saccade onset.",
     "found": null,
     "status": null
    },
    {
     "kind": "metadata_insufficient",
     "statement": "The electrodes tables of all spikesorting files give location 'unknown'; the brain area of each probe (s0 in DMFC, vprobe0 and vprobe1 in FEF) is stated only in the paper's code and the claims, not in the dandiset.",
     "found": null,
     "status": null
    },
    {
     "kind": "selection_undocumented",
     "statement": "The dandiset holds two Perle sessions the paper does not count, 2022-06-06 (spike sorting, 2,378 trials, all with 2 objects, 77 units) and 2022-06-08 (behaviour only, 1,517 trials); neither the dandiset nor the paper says what they are. The paper's 22 Ring sessions include Elgar 2022-09-18 and 2022-09-19, which the code's Ring model comparison leaves out.",
     "found": null,
     "status": null
    },
    {
     "kind": "paper_inconsistency",
     "statement": "The paper's counts of 'well-isolated neurons' (4,688 Triangle, 3,051 Ring) are recovered only with multi-units included: 1,563 of the 4,688 Triangle units and 1,123 of the Ring units are labelled good (single units), the rest mua. The counts also do not apply the Methods' rule of at least 500 stable trials (4,473 with it).",
     "found": null,
     "status": null
    },
    {
     "kind": "code_mismatch",
     "statement": "The AUC p-value of the code (phys_processing/selectivity/auc.py) draws labels with replacement in value-sorted order 5,000 times, a resampled null, where the Methods shuffle labels 1,000 times; under the code's p-value more units pass the modeling threshold than under the Methods' (for Elgar 2022-08-20, 172 against 98).",
     "found": null,
     "status": null
    },
    {
     "kind": "methods_unavailable",
     "statement": "Question for the authors, fig2d-signal-noise-triangle: Fig. 2d's signal correlation is defined in the Methods as the correlation of the 9 1-object condition means, while figures/figure_2/noise_correlations/run_compute_triangle.py computes it per pair of 1-object conditions with shuffled trials. Which one does the figure show? With the Methods' definition the lowest bin is negative in both monkeys (P -0.0041, E -0.0230); with the code's, Monkey E's lowest bin holds 124 rows and is 0.0245 (interval -0.0296 to 0.0786).",
     "found": null,
     "status": null
    },
    {
     "kind": "methods_unavailable",
     "statement": "Question for the authors, ext1-misbinding-errors and correctness in general: is a response assigned to an object, and counted correct, within 2.4 degrees (Methods) or 4.8 degrees (code, and the reward in the data)? With 2.4 degrees, Monkey P's 2-object errors are mostly at no vertex (249 against 207 at the non-target object); with 4.8 degrees misbinding is the main error class in both monkeys. This attempt assumed 4.8 degrees.",
     "found": null,
     "status": null
    },
    {
     "kind": "methods_unavailable",
     "statement": "Question for the authors, fig2b-selectivity-range and fig5cd-ring-tuning: the Fig. 2b legend calls the AUC values cross-validated, but neither the Methods nor the code says how; this attempt used the uncross-validated AUC of the Methods.",
     "found": null,
     "status": null
    },
    {
     "kind": "code_mismatch",
     "statement": "The paper's 2-object decoder (figures/supp_fig_decoding/modeling/models.py, LinearModel2Objects) passes both decoders through a sigmoid, with outputs in 0 to 1, but for the Ring its targets (labelers.py, 2x - 1 and 2y - 1) lie in -0.7 to 0.7, so the negative half cannot be reached; the Methods describe linear decoders. With a linear output the Ring neural decoders here are far from orthogonal (median angles 140.2 and 142.1 degrees, against 88.8 and 93.2 with the sigmoid).",
     "found": null,
     "status": null
    },
    {
     "kind": "paper_inconsistency",
     "statement": "The Methods (Decoding Analyses) place the decoder-orthogonality angles in Extended Data Figure 3b (\"That is shown on the y-axis of Extended Data Figure 3b\"), while the Extended Data Fig. 3 legend gives them as panel (c), with the angle on the x-axis; panel b is the 2-object absent-feature decoding.",
     "found": null,
     "status": null
    }
   ],
   "reconciliation": null,
   "verifications": [
    {
     "date": "2026-11-04",
     "kind": "agent",
     "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)",
     "reviewed": "Round 1. The executed notebooks reproduce_watters_2026.ipynb and decoding_watters_2026.ipynb with the conclusion cells and the summaries removed, the helper module wm.py and the claims file, with the paper. The verifier checked the archive's file list for the blocked claims and tested helper functions on synthetic inputs; it did not have the paper's code. Its report had no attempt record and gave its own outcome for each claim; 25 of 26 agree with the record, and for ext3c-decoder-orthogonality it gave partially reproduced where the record had not reproduced.",
     "agreed": 25,
     "of": 26,
     "disagreements": [
      {
       "claim": "ext3c-decoder-orthogonality",
       "outcome": "partially_reproduced",
       "reason": "Receptive Field synthetic data have medians of 91.6 to 96.7 degrees, and Ring neural data 88.8 / 93.3; both readings hold for these. Triangle neural data have medians of 106.7 / 106.5, which fail both readings. The criterion requires both neural and Receptive Field data to pass. get_simultaneous cuts Triangle 2022-06-04 to 20 of 116 units and Ring 2022-06-09 to 44 of 89, where the Methods assume about 100 simultaneously recorded units, and the notebook prints no per-session medians. The Ring decoders' outputs are sigmoids in (0, 1) while the Ring targets lie in -0.7 to 0.7, and the helper does not show that models.py uses a sigmoid for the Ring. Input scaling (as for ext3ab) may also affect the Triangle angles."
      }
     ]
    },
    {
     "date": "2026-11-05",
     "kind": "agent",
     "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)",
     "reviewed": "Round 2. The full executed notebooks reproduce_watters_2026.ipynb, decoding_watters_2026.ipynb and decoding_variants_watters_2026.ipynb with their conclusions and summaries, the helper module wm.py, the claims file, the attempt record and the paper. The verifier agreed with all 26 outcomes; for ext3c-decoder-orthogonality it asked for assumed true, and it found prose numbers in the decoding notebooks, the cost cell and the record that the saved outputs do not support.",
     "agreed": 26,
     "of": 26,
     "disagreements": []
    }
   ]
  }
 ],
 "entries": [
  {
   "name": "000620-watters-2026",
   "type": "reproduction",
   "title": "Multi-object working memory in macaque frontal cortex (Watters et al. 2026)",
   "description": "Tests the findings of Watters, Gabel, Tenenbaum and Jazayeri (2026) on dandiset 000620, which holds the behaviour and spike-sorted units of 46 physiology sessions from two macaques in DMFC and FEF during a multi-object working-memory task (24 Triangle and 22 Ring sessions). It tests behaviour, unit counts, delay-phase selectivity, signal and noise correlations and the decoding analyses; the claims that rest on fitting the paper's Slot, Switching and Gain encoding models are blocked as too large to run here.",
   "datasets": [
    "dandi:000620"
   ],
   "papers": [
    "10.64898/2026.01.27.702062"
   ],
   "derived_from": [],
   "notebooks": [
    {
     "path": "reproduce_watters_2026.ipynb",
     "title": "Behaviour, units, selectivity and correlations on dandiset 000620",
     "executed": "notebooks/000620-watters-2026/reproduce_watters_2026.html"
    },
    {
     "path": "decoding_watters_2026.ipynb",
     "title": "Decoding analyses (Extended Data Fig. 3) on dandiset 000620",
     "executed": "notebooks/000620-watters-2026/decoding_watters_2026.html"
    },
    {
     "path": "decoding_variants_watters_2026.ipynb",
     "title": "Variants of the decoding analyses (input scale, Ring output) on dandiset 000620",
     "executed": "notebooks/000620-watters-2026/decoding_variants_watters_2026.html"
    }
   ]
  }
 ]
}