{ "dataset": { "archive": "crcns", "id": "ofc-4", "label": "CRCNS:ofc-4" }, "page": "datasets/crcns/ofc-4/", "generated": "2026-11-05T06:30:02+00:00", "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.", "outcome_counts": { "reproduced": 10, "partially_reproduced": 6, "not_reproduced": 5, "blocked": 8 }, "outcome_counts_with_outside_data": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_elsewhere": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_links": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "attempts": [ { "id": "crcns-ofc-4-schuck-2016-01", "dataset": { "archive": "crcns", "id": "ofc-4", "version": "unversioned", "doi": "10.6080/K0FN14CH" }, "key": "crcns:ofc-4", "label": "CRCNS:ofc-4", "dataset_version": "unversioned", "paper": "10.1016/j.neuron.2016.08.019", "paper_title": null, "date": "2026-11-04", "kind": "agent", "accountable": "CatalystNeuro", "agent": { "harness": "Claude Code", "model": "claude-opus-5-5", "protocol": "pilot-13" }, "runner": { "platform": "GitHub Actions", "url": "https://github.com/catalystneuro/encore/actions/runs/41889942655" }, "tier": "agent_verified", "deprecated": "", "deprecated_on": null, "entry": "crcns-ofc-4-schuck-2016", "entry_title": "Human orbitofrontal cortex represents a cognitive map of state space (Schuck et al. 2016)", "description": "Tests the findings of Schuck, Cai, Wilson and Niv (2016) on CRCNS data set ofc-4, which holds the OFC fMRI time series, design matrices and event files of 24 of the paper's 27 participants. It covers the behavioural results that the event files allow, the 16-state and alternative decoding analyses, the permutation control, the similarity structure of state patterns, the trial-wise decoding around errors and the relation of pattern similarity to errors. The whole-brain, other-ROI, training and simulation results cannot be tested on it.", "notebooks": [ { "path": "reproduce_schuck_2016.ipynb", "title": "Reproducing Schuck et al. (2016) from CRCNS ofc-4", "executed": "notebooks/crcns-ofc-4-schuck-2016/reproduce_schuck_2016.html" } ], "outcomes": [ { "claim": "behavior-error-rate", "statement": "Participants performed the task nearly without errors, with a mean error rate of 2.3% in the scanner.", "source": "Results, paragraph 4 (after Figure 1); Supplemental Results 1.1", "quote": "Participants performed the task with near optimal performance (mean error rate: 2.3%; Figure S1)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "metadata_insufficient", "notebook": "reproduce_schuck_2016.ipynb", "notes": "The event files do not record the number of errors. Each 'err' event is a block that starts at an error trial and runs over its repetitions and often the trials after it (spans 6.8 to 41 s). Counting one error per block gives a mean of 1.92% (all trials with one repetition per error) or 1.98% (96 trials per run), outside the 2.07 to 2.53% of the tolerance; counting every designed trial inside a block as an error gives 3.10%. The paper's 2.3% lies between these bounds, so the value cannot be decided. Time-outs cannot be told from errors, so the criterion's two readings on time-outs give the same count. The data set holds 24 of the 27 participants. Measured in trial slots, 82 of the 172 blocks after an Exit trial span about two (an error and its repetition) and most others three to five; a least-squares fit over runs gives 1.52 designed trials per block after an Exit trial and 1.87 per other block, so a block after an Exit trial holds on average about half a designed trial besides the error trial (0.52) and another block close to one (0.87); that trial may be a second error or a correct trial after the error. The 'err' RT-file duration runs to the last response of the block.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-errors-by-trial-class", "statement": "Error rates differed by trial class (Enter 3.4%, Exit 2.3%, Internal 1.4%).", "source": "Supplemental Results 1.1, paragraph 1", "quote": "but were affected by the class ... of the trial (Enter: 3.4%, Exit: 2.3%, Internal: 1.4%", "statistics": [ { "test": "mixed-effects model, Wald chi-square", "statistic": "\u03c72(2) = 20.8", "p": 0.001, "p_relation": "<", "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "The class effect is present and the rates do not match. Error rates differ by class (linear mixed model chi2(2) = 63.8, p = 1.4e-14; logistic variant chi2(2) = 73.0) and the Enter rate is higher than the other two classes together in all 24 participants, but the Enter rate is 4.79% against 3.4 +/- 0.34. Errors after an Exit trial (Enter errors) are 181 of 225 error blocks after a run's first trial; the 44 others are Internal or Exit errors whose class the files do not give, so Exit and Internal together are 0.60%, and given all 44 errors one class reaches at most 1.18% (Exit) or 1.20% (Internal) (measured gives them all to Internal, the first assignment shown). Neither Exit (2.3%) nor Internal (1.4%) can reach its expected value, and their order cannot be told. One error is counted per block, a lower bound, so the Enter rate can only be higher. Blocks after an Exit trial are 79% of blocks, where the paper's class rates imply that Enter errors are 48% of errors, so Internal and Exit errors may be undercounted or merged into blocks. Changed from not_reproduced after verification round 1.", "deviations": [ "Assumed that each block holds one error and that its class follows from the trial before the block (Enter after an Exit trial, Methods, Task); the files do not record either or, where every block of a run holds one designed trial, from the reconstructed state; the paper had the trial's own label.", "The logistic reading (b) uses participant fixed effects, as statsmodels has no frequentist logistic mixed model; reading (a) uses statsmodels' MixedLM in place of lmer." ], "assumed": true, "disagreed": true, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-errors-stable-over-blocks", "statement": "Error rates did not change over the blocks of the main experiment.", "source": "Supplemental Results 1.1, paragraph 1", "quote": "Behavioral error rates were 2.3% during the main experiment, with no evidence for error rates ... changing over time (main effect Block", "statistics": [ { "test": "mixed-effects model, main effect of block, Wald chi-square", "statistic": "\u03c72(1) = 0.08", "p": 0.77, "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Run effect on per-run error rates, random intercept and slope by participant, slope -0.11 percentage points per run, chi2(1) = 1.44, p = 0.23 (paper chi2(1) = 0.08, p = .77); logistic variant p = 0.22. Mean rates per run 1.94, 2.35, 2.27, 2.01, 1.51%.", "deviations": [ "Assumed, as for supp-errors-by-trial-class and fig4a-decoding-error-correlation, that each error block holds one error, a lower bound (see behavior-error-rate); the files do not record the number.", "statsmodels' MixedLM in place of lmer; the logistic variant uses participant fixed effects." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-errors-not-by-age-category", "statement": "Error rates were not affected by the age or the category judged, nor by their interaction.", "source": "Supplemental Results 1.1, paragraph 1", "quote": "Error rates were not affected ... by factors Age, Category or their interaction (all p\u2019s > .13)", "statistics": [ { "test": "mixed-effects model, Wald chi-square", "p": 0.13, "p_relation": ">", "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "On Enter trials whose current age is known (labelled Enter trials and 98 reconstructed Enter errors), category p = 0.41, age p = 0.63, interaction p = 0.95 (logistic variant p = 0.44, 0.61, 0.88). The category alone, over all trials and the 224 errors whose category is known, p = 0.23 (faces 1.82%, houses 2.22%).", "deviations": [ "The current age of an error trial is known only for Enter errors whose block spans two trials (error and repetition), under the span reading of `ofc4.error_trials`; the test of age and the interaction is therefore made on Enter trials only. This reading is an assumption about the content of the error blocks, which the files do not describe." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-training-errors-higher", "statement": "Errors were higher during training than in the main experiment in the scanner.", "source": "Supplemental Results 1.1, paragraph 1", "quote": "errors were significantly higher during ... training than during the main experiment, t(26) = 5.6, p < .001", "statistics": [ { "test": "t-test", "statistic": "t(26) = 5.6", "p": 0.001, "p_relation": "<", "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "The data set holds only the five scanner runs; the training session (three blocks of 97 trials outside the scanner) is not in it (zip listing and data descriptor), nor named anywhere else by the paper.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-training-rt-unchanged", "statement": "Reaction times did not change between training and the main experiment (991 against 985 ms).", "source": "Supplemental Results 1.1, paragraph 1", "quote": "not change between training and main experiment (991 vs 985 ms, t(26) = 0.27, p = .79)", "statistics": [ { "test": "t-test", "statistic": "t(26) = 0.27", "p": 0.79, "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "No training session in the data set, as for supp-training-errors-higher.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-rt-stable-over-blocks", "statement": "Reaction times did not change between blocks of the main experiment.", "source": "Supplemental Results 1.1, paragraph 1", "quote": "nor ... between blocks within the main experiment (\u03c72(1) = 1.7, p = .18)", "statistics": [ { "test": "mixed-effects model, Wald chi-square", "statistic": "\u03c72(1) = 1.7", "p": 0.18, "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Per participant and run, median RT of correct trials; slope -8.6 ms per run, chi2(1) = 1.74, p = 0.19 (paper chi2(1) = 1.7, p = .18).", "deviations": [ "The reading \"all responded trials\" cannot be run, as error trials have no RT in the files (an error block's duration is its span); correct trials only.", "RTs are given to 0.1 s in the event files." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-rt-faces-faster", "statement": "Reaction times were faster for faces than for houses (944 against 1028 ms), with no effect of age and no interaction.", "source": "Supplemental Results 1.1, paragraph 1", "quote": "by age (p = .67), but were faster for faces than for houses (944 vs 1028 ms", "statistics": [ { "about": "category", "test": "mixed-effects model, Wald chi-square", "statistic": "\u03c72(1) = 78.3", "p": 0.001, "p_relation": "<" }, { "about": "age", "test": "mixed-effects model, Wald chi-square", "p": 0.67 }, { "about": "age by category interaction", "test": "mixed-effects model, Wald chi-square", "p": 0.98 } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Mean over participants of category medians, faces 945.8 and houses 1012.5 ms, both within tolerance; category chi2(1) = 58.7, p = 1.8e-14; age p = 0.72; interaction p = 0.61. With by-participant category slopes added (a variant), category chi2(1) = 54.1, p = 1.9e-13. Pooled trial medians 900 and 1000 ms. With RTs given to 0.1 s, 15 of 24 participants have a lower face median.", "deviations": [ "Correct trials only; RTs to 0.1 s." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-rt-by-trial-class", "statement": "Reaction times differed by trial class, slightly faster on repeat (Internal) trials than on the other classes.", "source": "Supplemental Results 1.1, paragraph 1", "quote": "RTs were also affected by trial ... class with slightly faster trials in Repeat than in the other trial classes", "statistics": [ { "test": "mixed-effects model, Wald chi-square", "statistic": "\u03c72(2) = 153.7", "p": 0.001, "p_relation": "<" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Class effect chi2(2) = 116.6, p = 4.8e-26 (paper 153.7); mean of participant medians Internal 883, Enter 1021, Exit 1023 ms; Internal lowest in 21 of 24 participants.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2a-16way-decoding", "statement": "The 16 task states can be decoded from multivoxel patterns in anatomically defined OFC above chance (mean 12.2% against 6.25%).", "source": "Results, paragraph 5; Figure 2A", "quote": "we could decode significantly above chance in which of the 16 possible task-states participants were during the task (mean classification accuracy 12.2%, i.e., almost twice the 6.25% chance baseline", "statistics": [ { "test": "t-test against chance", "statistic": "t26 = 9.04", "p": 0.001, "p_relation": "<", "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "16-way accuracy 8.93% with maps z-scored by voxel (t23 = 3.18, one-sided p = 0.0021, two-sided p = 0.0041) and 8.92% by map (one-sided p = 0.00037), above chance but outside the 10.98 to 13.42% of the tolerance. With state regressors rebuilt from the event files, 9.57% and 9.19%. Above chance in 15 and 16 of 24 participants. Under the two readings of O3, rebuilt from the error blocks, 8.97% and 9.38% (voxel) and 8.37% and 8.10% (map), all above chance (one-sided p <= 0.0045) and below tolerance.", "deviations": [ "No spatial smoothing of the beta maps (O1); the files hold no voxel coordinates.", "The data set's GLM nuisance set (four drift terms, six motion parameters) without the paper's 26 cardiac regressors (O5), and the data set's AFNI design matrices in place of SPM8's.", "The data set's 'err' column follows a short event at each block onset, so with its columns the trials inside a block after the error trial are in no regressor, neither reading of O3. Both readings are rebuilt from the error blocks, (a) the 'err' column with a separate boxcar from onset + 4.5 s (the mean trial duration, Methods, Task) to the end of the block, (b) one 'err' boxcar over the block, and give the same outcome.", "24 of the 27 participants." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2-two-back-state-space", "statement": "A classifier for an alternative 16-state space built from the task-irrelevant category and age of two trials back decodes worse than the hypothesized state space (8.1% against 12.2%).", "source": "Results, paragraph 5", "quote": "Training and testing a classifier on this state space resulted in lower 16-way classification compared to the hypothesized state space (8.1% versus 12.2%, t26 = 2.8, p = 0.01)", "statistics": [ { "test": "t-test", "statistic": "t26 = 2.8", "p": 0.01, "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "The two-back space decodes worse than the hypothesized one, but no reading meets both parts of the criterion. With maps z-scored by voxel, it is lower (5.96% against 8.93%, t23 = -3.75, p = 0.001; reading b 6.17%, p = 0.003; against the hypothesized space rebuilt from the events, 9.57%, p = 6e-5 and 0.0003), but at chance, below the 7.29 to 8.91% of the tolerance. With maps z-scored by map, 7.88% in both readings, within tolerance, but not significantly lower (p = 0.13 and 0.14; 0.070 and 0.073 against the rebuilt space). 334 (a) and 295 (b) of 11071 labelled trials were left out as nuisance. Changed from not_reproduced after verification round 1.", "deviations": [ "Two-back regressors are rebuilt from the RT event files with SPM's canonical HRF, shifted by -0.6 s, at which rebuilt state columns match the data set's (r 0.983 to 1.0).", "Same unsmoothed maps and nuisance set as fig2a-16way-decoding." ], "assumed": false, "disagreed": true, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2b-hidden-components", "statement": "Within the 16-way classification, the three hidden, task-relevant components (previous category, previous age, current category) are each classified above chance.", "source": "Results, paragraph 5; Figure 2B", "quote": "In contrast, the category and age from one trial ago, as well as the current category\u2014all hidden and task-relevant components\u2014were classified above chance", "statistics": [ { "about": "previous category", "test": "t-test against chance, one-sided", "statistic": "t26 > 3", "p": 0.014 }, { "about": "previous age", "test": "t-test against chance, one-sided", "statistic": "t26 > 3", "p": 0.003 }, { "about": "current category", "test": "t-test against chance, one-sided", "statistic": "t26 > 3", "p": 0.007 } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Previous category 52.03% (one-sided t-test p = 0.048, bootstrap p = 0.046) is above 50%; previous age 49.08% (p = 0.77) and current category 50.26% (p = 0.42) are not. With maps z-scored by map, 51.78% (p = 0.038), 51.16% (p = 0.16) and 51.78% (p = 0.10). One of three components under both readings. Under the two readings of O3, previous category reaches p < 0.05 in three of four combinations (p = 0.013 to 0.034; 0.12 for O3 a by voxel), previous age and current category in none (p >= 0.10).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2b-current-age-absent", "statement": "The observable component, current age, is not classified above chance within the 16-way classification.", "source": "Results, paragraph 5; Figure 2B", "quote": "the analysis revealed no evidence that the current age, which was task-relevant but observable given the current category, contributed to state representations in the OFC (rightmost bar in Figure 2B, p > 0.72", "statistics": [ { "test": "test against chance, one-sided (test not named)", "p": 0.72, "p_relation": ">" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Current age 50.83% (one-sided p = 0.27) with maps z-scored by voxel, 47.92% (p = 0.95) by map; bootstrap p = 0.27 and 0.97. Under the two readings of O3, p = 0.094 to 0.86.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2b-two-back-absent", "statement": "The task-irrelevant category and age from two trials back are not classified above chance by the two-back classifier.", "source": "Results, paragraph 5; Figure 2B", "quote": "This analysis confirmed that no information about irrelevant events from two trials ago could be detected (two leftmost bars in Figure 2B, p > 0.66)", "statistics": [ { "test": "test against chance, one-sided (test not named)", "p": 0.66, "p_relation": ">" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Two-back category and age near 50% in every reading (48.53 to 51.42%); smallest one-sided t-test p = 0.055 (two-back category, maps z-scored by map, reading a). The bootstrap variant gives p = 0.042 for that component (0.081 in reading b), and above 0.5 by voxel.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "figs2c-8way-decoding", "statement": "Decoding an 8-state space without current age (previous category, previous age, current category) is above chance (16.7%), and its three components are decoded above baseline (53.5%, 52.9%, 55.5%).", "source": "Results, paragraph 6; Supplemental Results 1.2, paragraph 2; Figure S2C", "quote": "Decoding eight different states in the OFC in this manner, we indeed found significant decoding accuracy, albeit at a slightly lower level relative to chance baseline (16.7%, corresponding to 4.2% above chance baseline, t26 = 3.3, p = 0.002); all components involved in this reduced state space also showed above baseline decoding (53.5%, 52.9%, and 55.5%", "statistics": [ { "about": "8-way accuracy against chance, main text", "test": "t-test against chance", "statistic": "t26 = 3.3", "p": 0.002, "n": "27 participants" }, { "about": "8-way accuracy against chance, Supplemental Results 1.2", "test": "t-test against chance", "statistic": "t26 = 6.6", "p": 0.001, "p_relation": "<" }, { "about": "previous category", "test": "t-test against chance", "p": 0.03 }, { "about": "previous age", "test": "t-test against chance", "p": 0.09 }, { "about": "current category", "test": "t-test against chance", "p": 0.001, "p_relation": "<" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "With maps z-scored by voxel, 8-way accuracy 15.89% (within tolerance; one-sided p = 0.006), previous category 55.36% (p = 0.0024) and current category 54.48% (p = 0.0036) above 50%; previous age 49.09% (p = 0.67), which matches neither the main text (above baseline) nor the supplement's p = .09. With maps z-scored by map, 14.79% (outside tolerance, p = 0.025) and no component above 50% (p = 0.20 to 0.44), so the outcome rests on the z-scoring.", "deviations": [ "Beta maps z-scored per voxel across the state maps of each run, assumed as the operation of the Princeton MVPA toolbox's zscore_runs, which the Methods name; the Methods say only that the maps were z-scored within each run (O2). The per-map reading is shown and does not reproduce.", "The 8-state regressors are the sums of the data set's two state columns that differ in current age, which equals modelling them as one event type (Figure S2C legend)." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "figs2a-permutation-control", "statement": "Classifiers trained on permuted labels decode at chance, and the real decoding is above the permutation results for the 16-way classification and each binary component.", "source": "Results, paragraph 7; Supplemental Results 1.2, paragraph 1; Figure S2A", "quote": "First, a permutation analysis showed at chance decoding levels and was significantly lower than decoding on the real data, both in the 16-way analysis as well as regarding the binary classification (see black bar in Figures 2A and S2A, all p < 0.025)", "statistics": [ { "test": "permutation test", "p": 0.025, "p_relation": "<" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Permuted classifiers average 5.98% (not above 6.25%), and real 16-way accuracy exceeds them (paired one-sided p = 0.001; by map 0.0003). No component meets p < 0.025; previous category comes closest (p = 0.033 and 0.031), then current category by map (p = 0.049). The 95th percentiles over participants' averaged permutation accuracies (reading a) are all within tolerance of the paper's; over pooled values (reading b) they are 10.0% and about 57.5%.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "synthetic-data-control", "statement": "Decoding on synthetic fMRI data with univariate stimulus-driven OFC activation and each participant's event timing is not above chance (6.8%).", "source": "Results, paragraph 7; Experimental Procedures, Synthetic fMRI Data and Noise Simulations", "quote": "This analysis also showed no above-chance decoding (mean accuracy across simulated \u2018\u2018participants\u2019\u2019 6.8%, p = 0.43 when compared to the chance baseline of 6.25%)", "statistics": [ { "test": "comparison with chance baseline", "p": 0.43 } ], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "metadata_insufficient", "notebook": null, "notes": "The simulation places a spherical activation at each participant's peak of decoding and smooths the data to the real data's smoothness, all in whole-brain space. The data set's time series are OFC voxels with no coordinates, so no sphere, searchlight peak or smoothness estimate can be built, and the whole-brain data are not in the data set. neuRosim is an R package and was not tried.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2c-enter-cluster", "statement": "In the similarity structure of OFC state patterns, pairs of states are more similar when both are Enter or both non-Enter states than when they differ.", "source": "Results, paragraph 9; Figure 2C", "quote": "indicates that pairs of activity patterns were indeed more similar if two states were both Enter or both non-Enter states (p < 0.01; paired t test comparing each subjects\u2019 average within-class versus between-class correlations)", "statistics": [ { "test": "paired t test", "p": 0.01, "p_relation": "<", "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Raw (or map z-scored, which gives the same correlations) beta maps, diagonal left out, within-class 0.0355 against between-class 0.0320, t23 = 2.82, p = 0.0097, 20 of 24 participants; diagonal in p = 0.0019. Maps z-scored by voxel give p = 0.052 without the diagonal and 0.0023 with it. 20% of correlations (50% after voxel z-scoring) are negative and set to NaN.", "deviations": [ "Unsmoothed maps; the dendrogram uses average linkage, a standard choice, and is shown only." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2d-enter-nonenter-decoding", "statement": "All three hidden components are decoded above chance even when the 16-way classifier is tested separately on Enter and on non-Enter trials.", "source": "Results, paragraph 9; Figure 2D", "quote": "a decoding analysis done separately for Enter and non-Enter trials that showed that even within these subsets of trials the 16-way classifier could successfully detect all three hidden state components (Figure 2D, all t26 > 2, p < 0.05", "statistics": [ { "test": "t-test against chance", "statistic": "t26 > 2", "p": 0.05, "p_relation": "<", "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Reading (a), the mean of Enter and non-Enter accuracies, previous category 52.03% (one-sided p = 0.048), previous age 49.08% (p = 0.77), current category 50.26% (p = 0.42); by map 51.78% (p = 0.038), 51.16% (p = 0.16), 51.78% (p = 0.10). In reading (b) at most one of six accuracies has p < 0.05. The same holds under the two readings of O3.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig3-searchlight-conjunction", "statement": "In whole-brain searchlight decoding, medial OFC is the only area where all three hidden components are decoded together, peaking in the left medial orbital gyrus at MNI [3, 44, -14].", "source": "Results, paragraph 10; Figure 3E; Table S1", "quote": "a voxel-wise conjunction analysis on the information maps of hidden state components confirmed that, across participants, only medial OFC activity carried complete information about all three hidden state components", "statistics": [ { "test": "voxel-wise conjunction, uncorrected", "statistic": "Zpeak = 3.14", "p": 0.01, "p_relation": "<", "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "The data set holds OFC voxels in native space only, with no coordinates; the whole-brain data in MNI space that the searchlight needs are not in it (zip listing, data descriptor).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "other-rois-incomplete", "statement": "Hippocampus, DLPFC and FFA/PPA code current and previous category but not previous age, and show neither complete nor task-specific encoding. Their decoding accuracy does not correlate with error rate in hippocampus or DLPFC but does in PPA/FFA, and in none of them does pattern similarity relate to transition error rates.", "source": "Results, paragraphs 10, 11 and 15; Figure S2D to S2F", "quote": "None of these areas showed either complete encoding of all hidden state components nor specific encoding of only task-relevant variables", "statistics": [ { "about": "correlation of decoding with error rate, PPA/FFA", "test": "Pearson correlation", "statistic": "r = -0.40, t25 = -2.2", "p": 0.03 }, { "about": "correlation of decoding with error rate, hippocampus and DLPFC", "test": "Pearson correlation", "p": 0.14, "p_relation": ">" }, { "about": "similarity and error rate, hippocampus, DLPFC, FFA/PPA", "p": 0.17, "p_relation": ">" } ], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "Hippocampus, DLPFC and PPA/FFA time series are not in the data set, which holds the OFC mask only.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4a-decoding-error-correlation", "statement": "Across participants, mean classification accuracy of the three hidden components in OFC correlates negatively with error rate.", "source": "Results, paragraph 11; Figure 4A; Figure S4", "quote": "participants\u2019 average error rates were significantlycorrelated with themean classificationaccuracyofallthree hidden state components", "statistics": [ { "test": "Pearson correlation", "statistic": "r = -0.58, t25 = -3.6", "p": 0.002, "ci": { "level": 0.95, "low": -0.8432, "high": -0.0169 }, "n": "27 participants" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "With maps z-scored by voxel, r = -0.268 (t22 = -1.30, p = 0.21; bootstrap 95% CI -0.717 to 0.338 with 10000 resamples, -0.700 to 0.327 with 1000), inside the paper's CI but outside the tolerance. With maps z-scored by map, r = -0.004, outside the paper's CI. Spearman rho -0.21. The error rate counts one error per block, a lower bound.", "deviations": [ "Voxel z-scoring assumed (see figs2c-8way-decoding).", "Percentile bootstrap intervals; the boot package's default interval type is not stated.", "24 participants, so t22 in place of t25." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4b-error-trial-drop", "statement": "Trial-wise 16-way decoding is lower on error trials (4.2%) than in the five trials preceding an error (11%).", "source": "Results, paragraph 12; Figure 4B", "quote": "incorrect responses were accompanied by worse classification\u2014the average 16-way classification in the five trials preceding an error was 11% whereas accuracy was only 4.2% on error trials (t23 = 2.2, p = 0.04", "statistics": [ { "test": "t-test", "statistic": "t23 = 2.2", "p": 0.04, "n": "24 participants" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "With maps z-scored by voxel, the first volume at or after onset + 4.8 s, and the span reading of error states (81 errors, 21 participants), five preceding trials 8.02% against error trials 1.59%, t20 = 1.99, p = 0.061; with the strict reading (37 errors, 16 participants) p = 0.18. Both values are outside tolerance. With maps z-scored by map, error trials decode better than the trials before them (7.70% against 5.67%). No reading gives p < 0.05. Every error trial with a state is an Enter trial, and correct Enter trials decode at 0.90% by voxel (Exit 11.79%, Internal 14.30%), as that classifier names an Enter state on 5.60% of labelled volumes where 50% would be unbiased, so the comparison with the mixed-class trials before an error is confounded by class.", "deviations": [ "Volumes unsmoothed (no coordinates), where the Methods smooth them at 3 mm.", "Only Enter errors can be given a state, from the neighbouring trials; the paper had every error trial's state." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4b-decline-before-errors", "statement": "Decoding accuracy declines linearly over the trials leading up to and including an error, and no such decline is seen in matched time courses without errors.", "source": "Results, paragraph 12; Figure 4B", "quote": "there was a negative linear effect of trial on decoding in the trials leading up to and including an error ... No such decrease was detected in matched decoding time-courses that did not include errors", "statistics": [ { "about": "linear effect of trial, error time courses", "test": "mixed-effects model, Wald chi-square", "statistic": "\u03c72(1) = 4.7", "p": 0.03 }, { "about": "condition by trial interaction", "test": "mixed-effects model, Wald chi-square", "statistic": "\u03c72(1) = 4.3", "p": 0.04 }, { "about": "linear effect of trial, non-error time courses", "test": "mixed-effects model, Wald chi-square", "p": 0.33 } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "With maps z-scored by voxel, the error slope over trials -5 to 0 is negative but not significant in any reading (smallest p = 0.32), nor is the interaction. With maps z-scored by map and the nearest volume, slope and interaction are significant but positive (span reading, matched courses a, 1.47 percentage points per trial, p = 0.039, interaction p = 0.030), the opposite of the claim. Position 0 is always an Enter trial and position -1 an Exit trial, so the course also reflects how each class decodes.", "deviations": [ "Most random-slope fits did not converge in statsmodels; the random-intercept model is reported in their place, as the notebook marks; one singular random-intercept fit is left out.", "As fig4b-error-trial-drop for smoothing and error states." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4b-error-vs-matched", "statement": "Decoding on the error trial is lower than on matched correct trials, and marginally lower on the trial before the error.", "source": "Results, paragraph 13; Figure 4B", "quote": "decoding in the error trial was significantly lower than decoding in the matched correct trials (trial \u2018\u20180\u2019\u2019 in the figure; t24 = 2.3, p = 0.01) and marginally lower in the trial prior to the error", "statistics": [ { "about": "trial 0", "test": "t-test", "statistic": "t24 = 2.3", "p": 0.01 }, { "about": "trial -1", "test": "t-test", "statistic": "t23 = 1.5", "p": 0.07 } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "Matched to their class, error trials do not decode worse. Every error trial with a state is an Enter trial, and correct Enter trials decode at 0.90% with maps z-scored by voxel, against 11.79% for Exit and 14.30% for Internal trials. Error trials 1.59% against 1.42% at position 0 of matched courses (a) with an Enter trial there (t20 = 0.18, one-sided p = 0.57), and against 0.88% on all correct Enter trials (p = 0.68); p >= 0.45 in every reading. The criterion's readings, which do not match class, differ. With maps z-scored by voxel and matched courses (a), error trials are lower (1.59% against 9.21%, t20 = -4.07, one-sided p = 0.0003; strict reading 2.08% against 8.93%, p = 0.005); with one matched course per error (b) p = 0.044 to 0.14; with maps z-scored by map the error trial is not lower (p >= 0.69). Trial -1 is not lower by voxel (11.59% against 8.81%, one-sided p = 0.70; paper p = 0.07). 21 participants, so t20 where the paper gives t24. With maps z-scored by voxel, reading (c) is at floor (1.59% against 1.42% and 0.88%, below the 6.25% chance level), because that classifier names an Enter state on 5.60% of labelled volumes, and could not show a drop. The reading that could is the map z-scoring, where labelled Enter trials decode at 8.02%, and there the error trial is not lower under the criterion's readings (p >= 0.69) or under (c) (p >= 0.45). Changed from reproduced after verification round 1.", "deviations": [ "Assumed a class-matched comparison, reading (c), position 0 against matched courses whose position 0 is an Enter trial. The criterion's matched courses (a) and (b) do not match class; the paper's error trials were of all three classes and its matched courses of the same mix, while here only Enter errors can be given a state, so only a class-matched comparison compares like with like. That the paper's matched courses had the class mix of its error trials is an inference; the Figure 4B legend says only \"seven consecutive trials with no behavioral error\".", "As fig4b-error-trial-drop for smoothing and error states." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig5-similarity-predicts-errors", "statement": "Across the eight Exit to Enter transitions, transitions between states whose OFC patterns are more correlated have lower error rates.", "source": "Results, paragraph 14; Figure 5B and 5C", "quote": "Results showed that transitions to \u2018\u2018nearer\u2019\u2019 states (i.e., transitions between pairs of states whose neural representations were more highly correlated) were associated with lower error rates", "statistics": [ { "about": "linear effect of correlation on error rate, eight transition types", "test": "linear regression", "statistic": "t6 = -3.23", "p": 0.02 }, { "about": "within-subject mixed-effects analysis", "test": "mixed-effects model, Wald chi-square", "statistic": "\u03c72(1) = 3.0", "p": 0.08 }, { "about": "Figure 5B, group average by ordinal position", "p": 0.04 }, { "about": "Figure 5C, within-subject correlations against zero", "p": 0.01, "p_relation": "<" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "No reading gives a negative regression of error rate on pattern correlation at p < 0.05. Over the eight transition types the slope is negative but not significant (p = 0.08 to 0.37); over ordinal positions it is positive where significant (strict errors, p = 0.049 raw and 0.039 voxel z-scored). Within-participant correlations average -0.010 to 0.157 (one-sided p >= 0.46); the mixed model gives p >= 0.24 (paper 0.08).", "deviations": [ "Transition error rates use only the Enter errors whose state is reconstructed (98 span, 39 strict); 83 Enter errors whose current age is unknown are left out.", "NaN correlations (negative ones) rank lowest in the ordinal reading, a choice the paper does not state." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "figs5-euclidean-distance", "statement": "The relation between pattern similarity and error rate holds with between-run Euclidean distances in place of correlations.", "source": "Results, paragraph 14; Supplemental Results 1.5; Figure S5", "quote": "we repeated this same analysis using Euclidean distances instead of correlations and obtained the same results (see Figure S5)", "statistics": [ { "about": "Figure S5B, group average by ordinal position", "p": 0.03 }, { "about": "Figure S5C, within-subject correlations against zero", "p": 0.01, "p_relation": "<" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_schuck_2016.ipynb", "notes": "With the span reading of error states, error rate rises with Euclidean distance over ordinal positions (raw maps t = 6.17, p = 0.0008; voxel z-scored t = 4.10, p = 0.0064) and within- participant correlations are above 0 (mean r 0.260, p = 0.003; 0.313, p = 0.0003). With the strict reading (39 errors in 35 participant-transitions, 16 participants) the raw-map result is not significant (p = 0.17 and 0.105), and with maps z-scored by map the relation is absent or reversed. The regression over transition types is not significant in any reading (p >= 0.33). The same analysis with correlations does not reproduce (fig5-similarity-predicts-errors), where the paper reports the same result for both. Errors take trials out of a state's regressor, and distance falls with the Enter state's correct-trial count (mean within-participant r -0.212, p = 0.013 raw; -0.258, p = 0.003 voxel z-scored); with that count held constant, distance and error rate are not related (mean partial r 0.072, one-sided p = 0.22; 0.076, p = 0.18). The paper's analysis has the same coupling, so the outcome follows the criterion, but the result may come from trial counts.", "deviations": [ "Rests on the span reading of error states, an assumption about the content of the error blocks.", "As fig5-similarity-predicts-errors for the error rates." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig5-synthetic-control", "statement": "The similarity and error-rate analysis on synthetic data with univariate stimulus-driven OFC activation finds no correlation with performance.", "source": "Results, paragraph 14", "quote": "These control analyses found no significant correlations with performance (p > 0.18 for all)", "statistics": [ { "p": 0.18, "p_relation": ">" } ], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "metadata_insufficient", "notebook": null, "notes": "Depends on synthetic-data-control, which is blocked for lack of voxel coordinates and whole-brain data.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-motor-action-not-in-ofc", "statement": "The current motor response, decoded as part of an alternative state space, is found in left motor and visual cortex, and not in an ROI analysis of the whole OFC.", "source": "Supplemental Results 1.3, paragraphs 1 and 2; Figure S3A", "quote": "the effect was detected only at a ... lenient threshold and was not confirmed in an ROI analysis of the whole OFC (p = .40)", "statistics": [ { "about": "OFC ROI decoding of states with motor response", "p": 0.4 } ], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "The motor-response state space needs each trial's correct response side, which needs the trial's response mapping; neither is in the event files or the run files, and the whole-brain part needs data the data set does not hold.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null } ], "findings": [ { "kind": "metadata_insufficient", "statement": "The 'err' events are blocks that start at an error trial and run over its repetitions and often the trials after it (spans 6.8 to 41 s), and the data descriptor does not say so. The number of errors, time-outs, error reaction times and the state of most error trials cannot be read from the files; the class is known for 181 of 228 blocks and the full state for 39 to 98.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "The 'err' column of the design matrices does not follow the error blocks' RT or stimulus durations (r 0.30 to 0.76) but a short event at each block onset (r 0.83 to 0.98); the descriptor says the design matrices are boxcars from stimulus onset to button press.", "found": null, "status": null }, { "kind": "metadata_insufficient", "statement": "The OFC time series carry no voxel coordinates or mask (each run file holds only OFC_timecourse, TR, condition_label, design_matrix_task, n_TR, n_voxel and nuisance_timecourse), so spatial smoothing, searchlights and any spatial analysis cannot be done, and voxels can be taken to be the same across runs only from their equal counts.", "found": null, "status": null }, { "kind": "data_missing", "statement": "The data set holds 24 of the paper's 27 participants and the OFC only. The training session, whole-brain data, the hippocampus, DLPFC and PPA/FFA regions, the cardiac regressors and each trial's response mapping and response side are not in it; five of the paper's claims are blocked for missing data.", "found": null, "status": null }, { "kind": "other", "statement": "Enter errors (after an Exit trial) are 181 of 225 error blocks after a run's first trial (79% of all blocks, where the paper's class rates imply 48%), an Enter error rate of 4.79% against at most 1.2% for the other two classes together, where the paper reports 3.4%, 2.3% and 1.4%.", "found": null, "status": null }, { "kind": "format_error", "statement": "In Subj-01 run 4 a 'HoHy' trial follows a 'HoHy' trial, which the task rules do not allow; every other pair of adjacent labelled trials chains.", "found": null, "status": null }, { "kind": "metadata_insufficient", "statement": "Three runs (Subj-11 run 3, Subj-18 runs 4 and 5) have no 'first' event or column, as their first trial was an error; the descriptor mentions only runs without an 'err' column.", "found": null, "status": null }, { "kind": "paper_inconsistency", "statement": "The Figure 1B legend says errors on Internal and Exit trials led to a repetition of the preceding trial, while the Methods (Task) say Internal and Enter trials were repeated and only Exit errors led to the preceding trial. The data show long blocks after Internal and Exit errors, consistent with the legend, but cannot settle it.", "found": null, "status": null }, { "kind": "methods_unavailable", "statement": "Question for the authors. How were the beta maps z-scored within each run (each voxel across the state maps, or each map across voxels) and with what kernel were they smoothed? The decoding results here change with the z-scoring (8-way decoding, the correlation with error rate and the error-trial comparison hold only with voxel-wise z-scoring), and the assumption made was the voxel-wise z-scoring of the Princeton MVPA toolbox's zscore_runs.", "found": null, "status": null }, { "kind": "methods_unavailable", "statement": "Question for the authors. How were errors counted, and how was the state of an error trial labelled, given that the shared event files merge each error with its repetitions? The attempt reconstructed the state of Enter errors from the neighbouring trials and counted one error per block.", "found": null, "status": null } ], "reconciliation": null, "verifications": [ { "date": "2026-11-04", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "Round 1. The executed notebook of the author run with the conclusion cells and the summary removed, the helper module and the claims file, with the paper. The verifier read the zip listing and tested helper functions on synthetic inputs. Its report gave its own outcome for each claim; 27 of 29 agree with the record as it then stood. Report in transcripts/crcns-ofc-4/verifier-1-report.md.", "agreed": 27, "of": 29, "disagreements": [ { "claim": "supp-errors-by-trial-class", "outcome": "partially_reproduced", "reason": "The class effect has p about 1e-14, and Enter is the highest class in 24 of 24 participants. Enter is 4.79% against 3.4 +/- 0.34. Exit and Internal stay below tolerance however the 44 unknown-class errors are split (Exit at most 1.18%, Internal at most 1.20%), and their order cannot be determined. Class rates inherit the one-per-block undercount (problem 2)." }, { "claim": "fig2-two-back-state-space", "outcome": "partially_reproduced", "reason": "No reading meets both conditions. Under voxel z-scoring the two-back accuracy is 5.96%, chance and outside tolerance, but lower than the hypothesized space at p = 0.001; under map z-scoring it is 7.88%, within tolerance, but not lower (p = 0.13). Problem 6, for Enter errors the trial two back as presented is the error trial itself, but reading (b) used the trial before the block. Problem 7, the two-back space is compared with the data set's columns, not with the rebuilt '16rebuilt' space, the cleaner pairing." } ] }, { "date": "2026-11-04", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "Round 2. The full executed notebook of the round-1 revision, with its conclusions and summary, the helper module, the claims file, the README and this record, with the paper. The verifier agreed with all 29 stated outcomes, checked the conclusions against the printed outputs and listed stale numbers, and reviewed its round-1 points. Report in transcripts/crcns-ofc-4/verifier-2-report.md.", "agreed": 29, "of": 29, "disagreements": [] } ] } ], "entries": [ { "name": "crcns-ofc-4-schuck-2016", "type": "reproduction", "title": "Human orbitofrontal cortex represents a cognitive map of state space (Schuck et al. 2016)", "description": "Tests the findings of Schuck, Cai, Wilson and Niv (2016) on CRCNS data set ofc-4, which holds the OFC fMRI time series, design matrices and event files of 24 of the paper's 27 participants. It covers the behavioural results that the event files allow, the 16-state and alternative decoding analyses, the permutation control, the similarity structure of state patterns, the trial-wise decoding around errors and the relation of pattern similarity to errors. The whole-brain, other-ROI, training and simulation results cannot be tested on it.", "datasets": [ "crcns:ofc-4" ], "papers": [ "10.1016/j.neuron.2016.08.019" ], "derived_from": [], "notebooks": [ { "path": "reproduce_schuck_2016.ipynb", "title": "Reproducing Schuck et al. (2016) from CRCNS ofc-4", "executed": "notebooks/crcns-ofc-4-schuck-2016/reproduce_schuck_2016.html" } ] } ] }