{ "dataset": { "archive": "openneuro", "id": "ds000228", "label": "OPENNEURO:ds000228" }, "page": "datasets/openneuro/ds000228/", "generated": "2026-11-05T07:41:23+00:00", "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.", "outcome_counts": { "reproduced": 15, "partially_reproduced": 8, "not_reproduced": 12, "blocked": 0 }, "outcome_counts_with_outside_data": { "reproduced": 1, "partially_reproduced": 1, "not_reproduced": 3, "blocked": 1 }, "outcome_counts_elsewhere": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 1 }, "outcome_counts_links": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "attempts": [ { "id": "openneuro-ds000228-richardson-2018-01", "dataset": { "archive": "openneuro", "id": "ds000228", "version": "1.1.1", "doi": "10.18112/openneuro.ds000228.v1.1.1" }, "key": "openneuro:ds000228", "label": "OPENNEURO:ds000228", "dataset_version": "1.1.1", "paper": "10.1038/s41467-018-03399-2", "paper_title": null, "date": "2026-10-28", "kind": "agent", "accountable": "CatalystNeuro", "agent": { "harness": "Claude Code", "model": "claude-opus-5-5", "protocol": "pilot-12" }, "runner": { "platform": "GitHub Actions", "url": "https://github.com/catalystneuro/encore/actions/runs/40723326297" }, "tier": "runs", "deprecated": "", "deprecated_on": null, "entry": "openneuro-ds000228-richardson-2018", "entry_title": "Development of the social brain from age three to twelve (Richardson et al. 2018)", "description": "Tests the findings of Richardson, Lisandrelli, Riobueno-Naylor and Saxe (2018) on OpenNeuro ds000228, which holds movie-viewing fMRI of 122 children aged 3 to 12 and 33 adults, with the paper's preprocessed images, artifact and CompCor regressors, group ROIs and behavioural scores. It covers the behavioural results, motion, inter-region correlations within and across the ToM and pain networks, reverse correlation of network timecourses, functional maturity, and the face-region analyses.", "notebooks": [ { "path": "reproduce_richardson_2018.ipynb", "title": "Reproducing Richardson et al. (2018) from OpenNeuro ds000228", "executed": "notebooks/openneuro-ds000228-richardson-2018/reproduce_richardson_2018.html" } ], "outcomes": [ { "claim": "participants", "statement": "The sample is 122 children aged 3.5 to 12 years (64 female) and 33 adults aged 18 to 39 (20 female); five further children were excluded for excessive motion. By age group there are 17, 14, 34, 23 and 34 children aged 3, 4, 5, 7 and 8 to 12 years.", "source": "Methods, Participants; Supplementary Table 1; Figure 2 legend", "quote": "One hundred twenty two 3.5\u201312-year-old children (M(s.d.) = 6.7 (2.3); 64 females) participated in the study", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "All nine counts match exactly and the mean child age is 6.71 years (s.d. 2.33). The age ranges of the five child groups match Supplementary Table 1. The five children excluded for motion are not in the dataset, so that part of the statement cannot be checked beyond the absence of any participant with one third or more artifact timepoints (maximum 48 of 168).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-table1-behavior-by-age", "statement": "Mean ToM score rises across age groups (3yo .54, 4yo .63, 5yo .73, 7yo .88, 8-12yo .96), and the false-belief groups (pass/inconsistent/fail) are 4/4/9 at 3 years, 3/7/4 at 4 years and 23/9/2 at 5 years.", "source": "Supplementary Table 1", "quote": "", "statistics": [], "kind": "finding", "inferred": "Numbers read from Supplementary Table 1, an image in the supplement PDF; no sentence states them.", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Mean ToM scores .517, .610, .724, .875 and .956 are within 10% of the paper's. The false belief groups by age are 4/4/9, 3/7/4 and 23/9/2, as in the paper.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "beh-tom-age", "statement": "Among children, ToM score (proportion correct) increases with age (Kendall tau = .66), and among the 3- to 5-year-olds DCCS summary score also increases with age (Kendall tau = .20).", "source": "Results, Behavioral results, paragraph 1; Figure 1a", "quote": "Performance on the ToM task (proportion correct) and DCCS were both positively correlated with age", "statistics": [ { "about": "ToM score against age", "test": "Kendall tau correlation test", "statistic": "rk(120) = .66", "p": 1e-05, "p_relation": "<", "n": "122 children" }, { "about": "DCCS summary score against age", "test": "Kendall tau correlation test", "statistic": "rk(62) = .20", "p": 0.049, "n": "64 children aged 3 to 5" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "ToM score against age, tau-b = .665, p = 5.4e-26 (n = 122). DCCS against age, tau-b = .198, p = .048 (n = 64; paper p = .049). Tau-c gives .654 and .191.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "beh-tom-dccs", "statement": "In 3- to 5-year-olds, ToM and DCCS scores are positively correlated controlling for age (partial Kendall tau = .19).", "source": "Results, Behavioral results, paragraph 1", "quote": "In the 3-year to 5-year-old subset of children who completed both measures, ToM and DCCS scores were positively correlated", "statistics": [ { "test": "partial kendall tau correlation test, controlling for age", "statistic": "rk(61) = .19", "p": 0.03, "n": "64 children aged 3 to 5" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Partial Kendall tau of ToM and DCCS given age = .186, p = .031 (n = 64).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "beh-fb-groups", "statement": "Among the 65 children aged 3 to 5, the false-belief composite gives 30 passers (15 female), 20 inconsistent performers (13 female) and 15 failers (6 female).", "source": "Results, Behavioral results, paragraphs 1 and 2; Methods, Explicit ToM task and false-belief composite score", "quote": "categorize these children as false-belief passers (5\u20136 FB questions correct; n = 30 (15 female)), inconsistent performers (3\u20134 FB questions correct; n = 20 (13 female)), and false-belief failers (0\u20132 FB questions correct; n = 15 (6 female))", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "30 passers (15 female), 20 inconsistent (13 female), 15 failers (6 female) in the deposited FB_Group of the 65 children under 6; recomputing the groups from FB_Composite with the Methods' cut-offs gives the same. FB_Group is also filled for 57 children aged 6 and over (54 pass, 3 inc), which the paper does not use.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "beh-fb-tom", "statement": "False-belief failers and inconsistent performers score lower than passers on the remaining ToM items (means .55, .57 and .75).", "source": "Results, Behavioral results, paragraph 2", "quote": "False-belief task failers and inconsistent performers did worse on the remaining ToM items than passers", "statistics": [ { "about": "Pass against Fail", "test": "Tukey HSD test of ToM*FB-Group ANOVA", "statistic": "diff = 1.2", "p": 5e-05, "p_relation": "<" }, { "about": "Pass against Inc", "test": "Tukey HSD test of ToM*FB-Group ANOVA", "statistic": "diff = 1.08", "p": 0.0001, "p_relation": "<" }, { "about": "Inc against Fail", "test": "Tukey HSD test of ToM*FB-Group ANOVA", "statistic": "diff = .16", "p": 0.8 }, { "test": "Kruskal-Wallis rank sum test", "statistic": "H(2) = 22.96", "p": 0.0001, "p_relation": "<", "n": "Pass 30, Inc 20, Fail 15" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Reading (a), the 18 non-false-belief items deposited as ToM Booklet-Matched-NOFB, gives means .546, .572, .753 and Kruskal-Wallis H = 22.95, p = 1e-05. Tukey on z-scored scores (O9a), Pass-Fail diff 1.24, p = 3.8e-05; Pass-Inc 1.08, p = 7.8e-05; Inc-Fail .16, p = .85, the paper's diffs. Reading (b), the full ToM score, gives means .451, .571, .792 (failers and passers out of tolerance), H = 41.5, and a significant Inc-Fail difference (p = .0073), so the claim holds under reading (a) only.", "deviations": [ "The paper does not define \"remaining ToM items\"; the criterion's two readings were run and reading (a) matches. participants.json describes ToM Booklet-Matched-NOFB as the 18 items without the six false-belief questions." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "beh-fb-age", "statement": "False-belief failers are younger than inconsistent performers and passers (mean ages 4.1, 4.8 and 5.2 years).", "source": "Results, Behavioral results, paragraph 2", "quote": "False-belief task failers were on average younger than passers and inconsistent performers", "statistics": [ { "about": "Pass against Fail", "test": "Tukey HSD test of Age*FB-Group ANOVA", "statistic": "diff = 1.4", "p": 1e-05, "p_relation": "<" }, { "about": "Inc against Fail", "test": "Tukey HSD test of Age*FB-Group ANOVA", "statistic": "diff = .83", "p": 0.01 }, { "about": "Pass against Inc", "test": "Tukey HSD test of Age*FB-Group ANOVA", "statistic": "diff = .59", "p": 0.047 } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Mean ages 4.08, 4.75, 5.23. Tukey on z-scored age, Pass-Fail diff 1.42, p = 4.3e-06; Inc-Fail .83, p = .015; Pass-Inc .59, p = .047.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "beh-fb-dccs", "statement": "False-belief failers have lower DCCS scores (worse response inhibition) than the other two groups (means 1.73, 2.26, 2.33 for fail, inconsistent, pass), significantly against passers and marginally against inconsistent performers.", "source": "Results, Behavioral results, paragraph 2", "quote": "failers demonstrated worse response inhibition than the other two groups", "statistics": [ { "about": "Pass against Fail", "test": "Tukey HSD test of DCCS*FB-Group ANOVA", "statistic": "diff = .88", "p": 0.01 }, { "about": "Inc against Fail", "test": "Tukey HSD test of DCCS*FB-Group ANOVA", "statistic": "diff = .78", "p": 0.052 }, { "test": "Kruskal-Wallis rank sum test", "statistic": "H(2) = 7.56", "p": 0.02, "n": "Pass 30, Inc 19, Fail 15" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Mean DCCS 1.733, 2.263 (n = 19), 2.333; Kruskal-Wallis H = 7.56, p = .023; Tukey on the z scale, Pass-Fail diff .88, p = .012; Inc-Fail .78, p = .052, as in the paper.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "motion-child-adult", "statement": "Children have more artifact timepoints than adults (mean 10.5 against 2.8).", "source": "Methods, fMRI data analysis, paragraph 2; Supplementary Figure 8a", "quote": "Number of artifact timepoints differed significantly between child and adult participants", "statistics": [ { "test": "Welch two-sample t-test", "statistic": "t(137.7) = 6.49", "p": 1e-06, "p_relation": "<", "n": "122 children, 33 adults" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Artifact timepoints are the deposited ART indicator rows (one per flagged timepoint). Children 10.51 (s.d. 10.6), adults 2.82 (s.d. 4.0); Welch t = 6.49, df = 137.7, p = 1.4e-09.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "motion-not-age-tom", "statement": "Among children, the number of artifact timepoints is not correlated with age or with ToM score.", "source": "Methods, fMRI data analysis, paragraph 2", "quote": "Among children, number of motion artifact timepoints was not correlated with age ... or ToM score", "statistics": [ { "about": "artifact timepoints against age", "test": "spearman correlation test", "statistic": "rs(120) = .02", "p": 0.86, "n": "122 children" }, { "about": "artifact timepoints against ToM score", "test": "kendall tau correlation test", "statistic": "rk(120) = -.005", "p": 0.94, "n": "122 children" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Spearman with age .017, p = .86; Kendall with ToM score -.005, p = .94. Neither is significant.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "motion-not-fb", "statement": "In 3- to 5-year-olds, the number of artifact timepoints does not differ between false-belief passers and failers, nor vary with DCCS score.", "source": "Methods, fMRI data analysis, paragraph 2", "quote": "Number of artifact timepoints did not differ between young (3\u20135-year old) children based on falsebelief task performance", "statistics": [ { "about": "Pass against Fail", "test": "linear regression, effect of FB-group on number of artifact timepoints", "statistic": "b = -.04, t = -.12", "p": 0.9, "n": "Pass 30, Fail 15" }, { "about": "DCCS", "test": "linear regression, effect of DCCS summary score on number of artifact timepoints", "statistic": "b = .16, t = 1.18", "p": 0.25, "n": "64 children" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "FB group (fail coded 1, n = 45), t = .12, p = .90, z-scaled b = .039; DCCS (n = 64), t = 1.17, p = .24, z-scaled b = .148. Neither is significant.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-adult", "statement": "In adults, within-ToM (mean z .48) and within-Pain (.35) correlations are strongly positive and the across-network correlation is negative (-.17), each within-network value exceeding the across-network value.", "source": "Results, Inter-region correlation analysis, paragraph 2; Figure 2", "quote": "In adults, each network exhibited strong positive correlations within-network, and strong negative correlations across network", "statistics": [ { "about": "within-ToM against across-network", "test": "paired sample two-tailed t-test", "statistic": "t(32) = 19.1", "p": 2.2e-16, "p_relation": "<", "n": "33 adults" }, { "about": "within-Pain against across-network", "test": "paired sample two-tailed t-test", "statistic": "t(32) = 23.2", "p": 2.2e-16, "p_relation": "<", "n": "33 adults" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Main reading (O1 all voxels of the deposited masks, O2a, O3a). Adult means within-ToM .604, within-Pain .398, across -.051; paired t within-ToM against across 19.1 and within-Pain against across 21.8, both p < 1e-18. The two t values are in tolerance under three of the four O2 and O3 readings; the three means are out of tolerance under all. O1 with only voxels with data gives identical values. Variants outside the paper's method (no high-pass, 50 s high-pass, no CompCor) give adult means from .556 to .645 within-ToM and -.051 to .059 across, none near the paper's. Further diagnostic variants, adults (within-ToM, within-Pain, across), are mean r in place of z .519, .354, -.049; the high-pass basis in the nuisance regression .593, .393, -.086; left and right M1 as two regressors .599, .390, -.037; and the mean ToM and Pain signal as an added regressor .286, .258, -.382. Only the last, which is not the paper's method, moves the across value past the paper's, and it also takes both within-network values well below the paper's (.286 and .258 against .48 and .35). A shared signal removed in the paper's processing would shift the across value in this direction, but this variant does not match the paper's pattern, and no step of the Methods accounts for the offset. The ROI masks are in MNI mm (9 mm sphere centroids within 3.7 mm of the stated centers, every voxel within 8.9 mm), and the images are int16 with no scaling. Fig. 2 shows the mean matrices only as colours, so the offset cannot be traced pair by pair; the adult and 3yo mean matrices are printed in the notebook.", "deviations": [ "The 100 s high-pass filter is SPM's discrete cosine basis (spm_filter); the Methods say SPM8 and \"a cutoff of 100 s\" without naming the filter. A standard choice.", "The cleaning is applied to the ROI mean timecourse, which equals cleaning each voxel and averaging, as every step of Methods paragraph 4 is linear and the same for each voxel." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-age-within", "statement": "Among children, within-ToM and within-Pain correlations increase with age, controlling for motion.", "source": "Results, Inter-region correlation analysis, paragraph 3; Figure 2", "quote": "Among children, within-ToM and within-Pain network correlations increased significantly with age", "statistics": [ { "about": "within-ToM", "test": "Spearman partial correlation test, motion as covariate", "statistic": "rs(119) = .37", "p": 5e-05, "p_relation": "<", "n": "122 children" }, { "about": "within-Pain", "test": "Spearman partial correlation test, motion as covariate", "statistic": "rs(119) = .28", "p": 0.002, "n": "122 children" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Partial Spearman with age, motion as covariate, n = 122 children. Within-ToM rs = .306, p = .00064; within-Pain .461, p = 1.1e-07 (main reading); .306 to .324 and .458 to .470 under the other readings. Both rise with age and are significant at .05, but both are out of tolerance (.37 and .28), and within-ToM does not reach the paper's bound p < .00005 under any reading, which Rule S counts as a failed test.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-age-across", "statement": "Among children, the across-network correlation decreases with age (becomes more anti-correlated), controlling for motion.", "source": "Results, Inter-region correlation analysis, paragraph 3; Figure 2", "quote": "Across-network correlations decreased significantly with age", "statistics": [ { "test": "Spearman partial correlation test, motion as covariate", "statistic": "rs(119) = -.35", "p": 0.0001, "p_relation": "<", "n": "122 children" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Across-network partial rs with age = -.259, p = .0041 (main reading; -.259 to -.271, p = .0026 to .0041 under the others). It falls with age, significant at .05, but is out of tolerance (-.35) and does not reach the paper's bound p < .0001, which Rule S counts as a failed test.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-adult-vs-child", "statement": "Adults have higher within-ToM and within-Pain correlations and a more negative across-network correlation than children, controlling for motion.", "source": "Results, Inter-region correlation analysis, paragraph 3", "quote": "Within and across-network correlations were significantly greater in adults, compared to children", "statistics": [ { "about": "within-ToM, effect of group (child against adult)", "test": "linear regression with age group and motion", "statistic": "b = -.97, t = -5.7", "p": 6.2e-08, "p_relation": "<", "n": "122 children, 33 adults" }, { "about": "within-Pain, effect of group", "test": "linear regression with age group and motion", "statistic": "b = -.75, t = -3.8", "p": 0.0002, "n": "122 children, 33 adults" }, { "about": "across-network, effect of group", "test": "linear regression with age group and motion", "statistic": "b = 1.26, t = 7.2", "p": 2.2e-11, "n": "122 children, 33 adults" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Regression on group (child = 1) and artifact count. Child effect on within-ToM t = -6.54, p = 8.7e-10; within-Pain t = -6.72, p = 3.5e-10; across t = 4.49, p = 1.4e-05; all with the paper's signs and below its p values, but the t values are out of tolerance, within-Pain (-6.48 to -6.72) and across (3.92 to 4.49) under every reading, within-ToM under three of the four O2 and O3 readings (-6.54 to -7.05; -6.23 under O2a with O3b is within -5.7 +/- .57). z-scaled b (O4a) -1.16, -1.20, .83; raw b -.173, -.135, .087. Motion effects differ from the paper's, within-ToM t = -1.64, p = .10 (paper -4.3), across t = 2.87, p = .005 (paper not significant).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-control-networks", "statement": "Without M1 regression, within-ToM (rs .39), within-Pain (.38), within-Scene (.23) and within-M1 (.30) correlations rise with age in children, within-Face and within-V1 do not, and the across ToM-Pain correlation falls with age (-.26) while across Face-Scene and M1-V1 do not change.", "source": "Supplementary Figure 3 legend; Results, Inter-region correlation analysis, paragraph 3", "quote": "Significant positive age correlations among children (n=122) are present for withinToM (rs=.39, p<.0001), within-Pain (rs=.38, p<.0001), within-Scene (rs=.23, p=.01), and withinM1 regions", "statistics": [ { "about": "within-ToM", "test": "spearman partial correlation test, motion as covariate", "statistic": "rs = .39", "p": 0.0001, "p_relation": "<" }, { "about": "within-Pain", "test": "spearman partial correlation test, motion as covariate", "statistic": "rs = .38", "p": 0.0001, "p_relation": "<" }, { "about": "within-Scene", "test": "spearman partial correlation test, motion as covariate", "statistic": "rs = .23", "p": 0.01 }, { "about": "within-M1", "test": "spearman partial correlation test, motion as covariate", "statistic": "rs = .30", "p": 0.001 }, { "about": "within-Face", "test": "spearman partial correlation test, motion as covariate", "statistic": "rs = .06", "p": 0.53 }, { "about": "within-V1", "test": "spearman partial correlation test, motion as covariate", "statistic": "rs = -.16", "p": 0.09 }, { "about": "across ToM-Pain", "test": "spearman partial correlation test, motion as covariate", "statistic": "rs = -.26", "p": 0.005 }, { "about": "across Face-Scene", "test": "spearman partial correlation test, motion as covariate", "statistic": "rs = .02", "p": 0.8 }, { "about": "across M1-V1", "test": "spearman partial correlation test, motion as covariate", "statistic": "rs = .18", "p": 0.05 } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "No M1 regression, main reading. Age partial rs, within-ToM .367 (p = 3.5e-05), within-Pain .525 (p = 6.4e-10), within-Scene .288 (p = .0014), within-M1 .227 (p = .012), across ToM-Pain -.247 (p = .0064); within-ToM and across are in tolerance, within-Pain, Scene and M1 are not. Within-Face .106 (p = .25), within-V1 -.069 (p = .45) and across Face-Scene .065 (p = .48) are not significant, as in the paper; across M1-V1 is, .200 (p = .028; .017 to .034 under the other readings), where the paper has p = .05 and the criterion counts it as no change. The ToM and Pain results hold and one control result differs, the criterion's partial outcome.", "deviations": [ "The face and scene parcels are the ones deposited in derivatives/ROIs of the snapshot (rSTS_masked, lSTS, rOFA, lOFA, rFFA, lFFA; rRSC, lRSC, rTOS, lTOS, rPPA, lPPA), which the dataset's README names as the Julian et al. 2012 parcels; the claims file treats them as an outside input, but no outside data were used. The M1 and V1 ROIs are also deposited." ], "assumed": false, "disagreed": false, "data": "with_inputs", "where": null, "proposed": null }, { "claim": "irc-williams", "statement": "The age correlations of within-ToM and within-Pain are stronger than those of within-Face and within-V1, not stronger than within-Scene and within-M1, and the across ToM-Pain decline is stronger than across Face-Scene and across M1-V1.", "source": "Supplementary Figure 3 legend", "quote": "Positive correlations between within-ToM and within-Pain correlations and age were significantly stronger than within-Face and within-V1 correlations, but not significantly stronger than within-Scene and within-M1 correlations", "statistics": [ { "about": "within-Face against within-ToM", "test": "Williams' test of differences in age correlations", "statistic": "z = 2.69", "p": 0.01 }, { "about": "within-Face against within-Pain", "test": "Williams' test", "statistic": "z = 2.66", "p": 0.01 }, { "about": "within-Scene against within-ToM", "test": "Williams' test", "statistic": "z = 1.33", "p": 0.18 }, { "about": "within-Scene against within-Pain", "test": "Williams' test", "statistic": "z = 1.3", "p": 0.19 }, { "about": "M1 against within-ToM", "test": "Williams' test", "statistic": "z = .76", "p": 0.44 }, { "about": "M1 against within-Pain", "test": "Williams' test", "statistic": "z = .73", "p": 0.47 }, { "about": "V1 against within-ToM", "test": "Williams' test", "statistic": "z = 4.35", "p": 0 }, { "about": "V1 against within-Pain", "test": "Williams' test", "statistic": "z = 4.32", "p": 0 }, { "about": "across ToM-Pain against across Face-Scene", "test": "Williams' test", "statistic": "z(122) = 2.2", "p": 0.03 }, { "about": "across ToM-Pain against across M1-V1", "test": "Williams' test", "statistic": "z(122) = 3.39", "p": 0 } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Williams' T2 (and Steiger's Z, which agrees) on the age correlations of irc-control-networks. Under both readings (partial with motion, or plain Spearman) and every pipeline reading, the six differences the paper finds significant are significant in the same direction (main reading, reading a, t = 2.51 to 5.03 for Face and V1 against ToM and Pain; -2.95 and -4.0 for across ToM-Pain against Face-Scene and M1-V1). ToM against Scene (t = .75) and ToM against M1 (t = 1.32) are not significant, as in the paper, but Pain against Scene (t = 2.62, p = .0098) and Pain against M1 (t = 2.92, p = .0043) are, so the within-Pain age effect is stronger than Scene and M1, unlike the paper. 8 of 10 comparisons follow the paper under every reading. The criterion requires all ten and states no partial outcome, and two of the paper's tests (p = .19 and .47) come out on the other side of .05, which Rule S counts as failed tests, so the claim is not reproduced.", "deviations": [ "The paper reports z for Williams' test; Williams' T2 is a t on n - 3 df. Both T2 and Steiger's Z are shown; they give the same conclusions." ], "assumed": false, "disagreed": true, "data": "with_inputs", "where": null, "proposed": null }, { "claim": "irc-3yo-distinct", "statement": "In 3-year-olds, within-ToM (mean z .21) and within-Pain (.23) correlations are positive and higher than the across-network correlation, so the two networks are already distinct.", "source": "Results, Inter-region correlation analysis, paragraph 4; Figure 2", "quote": "Within-network correlations were higher than the acrossnetwork correlation", "statistics": [ { "about": "within-ToM against across-network", "test": "paired sample two-tailed t-test", "statistic": "t(16) = 6.2", "p": 5e-05, "p_relation": "<", "n": "17 three-year-olds" }, { "about": "within-Pain against across-network", "test": "paired sample two-tailed t-test", "statistic": "t(16) = 6.9", "p": 1e-05, "p_relation": "<", "n": "17 three-year-olds" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "17 three-year-olds, main reading. Within-ToM .326, within-Pain .179, both above the across-network value (t = 8.43, p = 2.8e-07; t = 4.54, p = .00034). The networks are distinct, but no value is in tolerance under any reading and the Pain test does not reach the paper's bound p < .00001 (p = .00013 to .00042 across readings), a failed test under Rule S.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-3yo-not-anticorrelated", "statement": "Unlike adults, 3-year-olds show no anti-correlation between the ToM and Pain networks (mean across-network z .05).", "source": "Results, Inter-region correlation analysis, paragraph 4; Figure 2", "quote": "unlike adults, ToM and pain networks were not anti-correlated in 3 year olds", "statistics": [], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "3yo across-network mean .074 (s.e. .020; .074 to .075 under every reading), not negative, just above the tolerance .05 \u00b1 .02. Variant one-sample t = 3.75, p = .0017, so the 3yo across correlation is above zero. Adults -.051.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-3yo-homologous", "statement": "In 3-year-olds the strongest within-network correlations are between homologous regions in opposite hemispheres, such as right and left TPJ and right and left insula.", "source": "Results, Inter-region correlation analysis, paragraphs 4 and 5; Figure 2 (3 years matrix)", "quote": "The strongest within-network correlations in the 3 year olds were between homologous pairs ... of regions in opposite hemispheres, such as right and left TPJ (ToM), and the right and left insula (Pain)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "In the 3yo mean matrix (main reading) the strongest Pain pair is RInsula-LInsula (.856) and the three homologous Pain pairs rank first to third (.856, .656, .339). Among ToM pairs RTPJ-PC (.637), LTPJ-PC (.635) and DMPFC-MMPFC (.609) are above RTPJ-LTPJ (.558), which ranks fourth, so neither reading holds for ToM. RTPJ-LTPJ ranks fourth and RInsula-LInsula first under all eight O1, O2 and O3 readings.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-3yo-nonbilateral", "statement": "In 3-year-olds, within-network correlations excluding bilateral pairs (ToM .20, Pain .17) still exceed the across-network correlation.", "source": "Results, Inter-region correlation analysis, paragraph 5", "quote": "However, the functional separation of the two networks was not fully explained by the strong correlations between bilateral pairs", "statistics": [ { "about": "within-non-bilateral-ToM against across-network", "test": "paired sample two-tailed t-test", "statistic": "t(16) = 5.1", "p": 0.0001, "n": "17 three-year-olds" }, { "about": "within-non-bilateral-Pain against across-network", "test": "paired sample two-tailed t-test", "statistic": "t(16) = 4.4", "p": 0.0005, "n": "17 three-year-olds" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Reading (b), dropping RTPJ-LTPJ and RInsula-LInsula, main pipeline reading. Non-bilateral ToM .309 (t = 7.64, p = 1e-06) and Pain .146 (t = 3.03, p = .008) both exceed across, with p on the paper's side of .05, but the means and t values are out of tolerance, except the Pain mean under O2b (.152, .153). Reading (a), dropping also RS2-LS2 and RMFG-LMFG, gives Pain .106, t = 1.34, p = .20, not significant. The criterion allows either reading; under (b) the tests hold and values differ.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-tom-behavior", "statement": "Among children, within-ToM correlation rises and across-network correlation falls with ToM score, controlling for motion.", "source": "Results, Inter-region correlation analysis, paragraph 6", "quote": "In children, the strength of inter-region correlations within the ToM network was positively correlated with behavioral performance on the ToM battery outside the scanner", "statistics": [ { "about": "within-ToM against ToM score", "test": "Kendall tau partial correlation test, motion as covariate", "statistic": "rk(119) = .23", "p": 0.0002, "n": "122 children" }, { "about": "across-network against ToM score", "test": "Kendall tau partial correlation test, motion as covariate", "statistic": "rk(119) = -.20", "p": 0.001, "n": "122 children" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Partial Kendall tau with ToM score, motion as covariate (ppcor form, O8). Within-ToM .246, p = 6.3e-05, in tolerance; across -.125, p = .042, out of tolerance (-.20 \u00b1 .02). Both have the paper's signs and p on the same side of .05. Other readings, .237 to .259 and -.121 to -.132.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-tom-behavior-age", "statement": "Controlling for age as well as motion, ToM score has no relation to within-ToM or across-network correlations.", "source": "Results, Inter-region correlation analysis, paragraph 6", "quote": "there was no relationship between within-ToM or across-network correlations and ToM score when controlling for age in addition to motion", "statistics": [ { "test": "linear regressions of within-ToM and across-network correlation on ToM score, age and motion", "statistic": "ts < 1", "p": 0.3, "p_relation": ">", "n": "122 children" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Regression on ToM score, age and motion, children. Main reading, ToM score effect on within-ToM t = 2.04, p = .044 (z-scaled b = .275), significant, where the paper has none; on across t = 1.29, p = .20. Readings, within-ToM p = .069 (O3b), .048 (O2b), .088 (O2b, O3b); across p = .19 to .27 in all. The outcome rests on the O2a and O3a readings.", "deviations": [ "Assumed O2a (one regressor per artifact timepoint), as deposited in the paper's own nuisance files, and O3a (all 168 TRs), since the Methods restrict the timepoints to TRs 11 to 168 only for the reverse correlation and maturity analyses (paragraphs 9 and 13), not for the inter-region correlation (paragraph 7). Under O3b the claim would be reproduced." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-fb-within", "statement": "Among 3- to 5-year-olds, false-belief passers have a higher within-ToM correlation than failers (.29 against .25), controlling for age and motion.", "source": "Results, Inter-region correlation analysis, paragraph 7; Figure 3a,b", "quote": "There was a significant difference in within-ToM network correlation between explicit false-belief task passers and failers", "statistics": [ { "test": "linear regression, effect of FB-Group (pass vs. fail) with age and motion", "statistic": "b = -.70, t = -2.06", "p": 0.046, "n": "Pass 30, Fail 15" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Passers (n = 30) .398, failers (n = 15) .345, so passers are higher, but the group effect with age and motion is t = .66, p = .51 (fail coded 1; z-scaled b = .26), not significant under any reading (p = .48 to .55). The paper has t = -2.06, p = .046 and means .29 and .25, both out of tolerance here.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-fb-within-dccs", "statement": "The passer and failer difference in within-ToM correlation becomes marginal when DCCS is added to the regression.", "source": "Results, Inter-region correlation analysis, paragraph 7; Figure 3 legend", "quote": "This group difference becomes marginal when response inhibition (DCCS summary score) is additionally included in the regression", "statistics": [ { "about": "FB-Group", "test": "linear regression with FB-Group, age, motion and DCCS", "statistic": "b = -.64, t = -1.80", "p": 0.079, "n": "Pass 30, Fail 15" }, { "about": "DCCS", "test": "linear regression with FB-Group, age, motion and DCCS", "statistic": "b = -.08, t = -.59", "p": 0.56 } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "With DCCS added, the group effect is t = .40, p = .69 (other readings p = .59 to .69), outside the .05 to .10 band; DCCS t = -.86, p = .39. It depends on irc-fb-within, which was tested and not reproduced.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "irc-fb-across", "statement": "False-belief passers and failers do not differ in across-network correlation (.04 against .03).", "source": "Results, Inter-region correlation analysis, paragraph 7; Figure 3b", "quote": "There was no difference in across-network correlation between these two groups", "statistics": [ { "test": "linear regression, effect of FB-group (pass vs. fail) with age and motion", "statistic": "b = .51, t = 1.2", "p": 0.23, "n": "Pass 30, Fail 15" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Passers .087, failers .057 (paper .04, .03). Group effect with age and motion t = -2.18, p = .035 under the main reading (fail coded 1, z-scaled b = -.87), significant, where the paper finds no difference. Readings, p = .051 (O3b), .034 (O2b), .047 (O2b, O3b). The outcome rests on the main reading.", "deviations": [ "Assumed O2a and O3a, as for irc-tom-behavior-age. Under O3b the claim would be reproduced (p = .051)." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "rc-events", "statement": "Reverse correlation on the adult network timecourses finds seven ToM events (68 s, mean length 9.7 s) and twelve pain events (86 s, mean length 7.2 s), at the times of Supplementary Table 3.", "source": "Results, Reverse correlation analysis, paragraph 2; Figure 4a; Supplementary Figure 5; Supplementary Table 3", "quote": "In adults, the reverse correlation analysis produced seven theory of mind events (68 s total, M(s.d.) length 9.7(4.2) s) and twelve pain events (86 s total, M(s.d.) length 7.2(4.7)s)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "The best-matching reading is O5a (p < .05 uncorrected); O6 changes nothing for the ToM and Pain networks. It finds 9 ToM events (80 s, mean 8.9 s) and 10 pain events (84 s, mean 8.4 s), covering all 7 ToM events and 10 of 12 pain events of Supplementary Table 3 (P08 and P10 missed). Under O5b and O5c, 7 ToM events (56 s, all 7 table events) and 8 pain events (62 s, 8 of 12). The pain time is in tolerance under O5a, the ToM time under none; the counts never both match. At least 5 of 7 and 9 of 12 table events are found under O5a, the partial outcome. Peak TRs of the found ToM events (131, 95, 64, 150, 46, 125, 54) are within one TR of the table's.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "rc-overlap", "statement": "ToM and pain events overlap at a single timepoint (the last of T05 and first of P05), less overlap than chance in a permutation test.", "source": "Results, Reverse correlation analysis, paragraph 2; Methods, fMRI data analysis, paragraph 10", "quote": "The timepoints that exceeded baseline for ToM and pain networks were almost entirely non-overlapping, with the exception of a single timepoint (2 s) ... This extent of overlap is significantly less than that that would occur by chance", "statistics": [ { "test": "random timecourse permutations with the same number and duration of events", "statistic": "5/1000 permutations with at most one timepoint of overlap", "p": 0.005 } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "The found events (O5a) overlap at no timepoint, against 1 in the paper; permutation p = .002 under reading (a), 158 TRs, and p < .001 (0 of 1000) under reading (b), 159 TRs (mean permuted overlap 11.9 TRs under both). Under O5b and O5c the overlap is also 0, p = .019 to .049. With the paper's own events (Supplementary Table 3, variant) the overlap is 1 TR, p = .002 and .008. Less overlap than chance holds; the overlap count does not match. The criterion states only when the claim is reproduced. The outcome is partial because the claim's test (p <= .01) holds and only its expected count is out of tolerance, the case the claims file's Rule S and the protocol call partial; irc-williams is not reproduced because two of the paper's tests fail there. The verifier would give not_reproduced in round 2; the disagreement is recorded under verifications.", "deviations": [ "Seed fixed (a standard choice); the random gaps split the remaining zeros by a multinomial draw after one zero between events, a reading of Methods paragraph 10, which says only \"a random proportion of zeros between events\".", "Reading (b) of the criterion is run as 159-TR timecourses (159 minus each network's event TRs as zeros), the length implied by the paper's zeros (125, 116) and event totals (34, 43 TRs); with the Supplementary Table 3 events it gives 125 and 116 zeros." ], "assumed": false, "disagreed": true, "data": "dataset", "where": null, "proposed": null }, { "claim": "rc-child-adult-timecourse", "statement": "The mean network timecourse of each child age group correlates with the adult mean timecourse (ToM r .28, .31, .60, .72, .82; Pain .60, .56, .73, .83, .89 for 3, 4, 5, 7 and 8-12 years).", "source": "Results, Reverse correlation analysis, paragraphs 3 and 4; Supplementary Table 4", "quote": "The average timecourse in ToM and pain regions in children was highly correlated with that of adults", "statistics": [ { "about": "ToM, every age bin", "test": "pearson correlation tests, TRs 11:168", "p": 0.0005, "p_relation": "<", "corrected": "Bonferroni, alpha .01 for five age bins" }, { "about": "Pain, every age bin", "test": "pearson correlation tests, TRs 11:168", "p": 1e-13, "p_relation": "<", "corrected": "Bonferroni, alpha .01 for five age bins" }, { "about": "ToM, 3yo", "test": "pearson correlation test", "statistic": "r = .28", "p": 0.00046 }, { "about": "Pain, 3yo", "test": "pearson correlation test", "statistic": "r = .60", "p": 1e-15, "p_relation": "<" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Every r is significant at the paper's Bonferroni alpha of .01 (ToM 3yo p = .0076, all others p < 3e-05); 8 of 10 are within 10%, ToM 3yo (.212 against .28) and Pain 3yo (.531 against .60) are below. Three correlations miss the paper's bounds, ToM 3yo p = .0076 against p < .0005 (the paper's exact p is .00046), Pain 3yo p = 7.1e-13 against p < 1e-15 and Pain 4yo p = 4.0e-12 against p < 1e-13. The criterion's partial outcome needs every r significant at the paper's bounds, so the claim is not reproduced. Judged only at the Bonferroni alpha it would be partially reproduced.", "deviations": [], "assumed": false, "disagreed": true, "data": "dataset", "where": null, "proposed": null }, { "claim": "supp-table4-other-networks", "statement": "The age-group mean timecourses of the Face, Scene, V1 and bilateral FFA regions correlate with the adult timecourse in every age group, and M1 does in all but the 3- and 4-year-olds.", "source": "Supplementary Table 4", "quote": "All correlations are significantly positive (ps<.0005) except those shaded in grey (3yo M1", "statistics": [ { "about": "all networks and age groups except 3yo and 4yo M1", "test": "Pearson correlation", "p": 0.0005, "p_relation": "<" }, { "about": "M1, 3yo", "test": "Pearson correlation", "statistic": "r = 0.11", "p": 0.16 }, { "about": "M1, 4yo", "test": "Pearson correlation", "statistic": "r = 0.06", "p": 0.46 } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "17 of 25 values within 10%, below the 20 the partial outcome needs. The significance pattern holds (only M1 3yo, p = .00078, and M1 4yo, p = .019, miss p < .0005), but all five M1 values are out of tolerance (.265, .186, .434, .350, .315 against .11, .06, .35, .44, .53), and the 3yo Face, V1 and FFA values are low. Scene matches in every group.", "deviations": [ "Face, scene, M1 and V1 ROIs are the ones deposited in the snapshot, not an outside input." ], "assumed": false, "disagreed": false, "data": "with_inputs", "where": null, "proposed": null }, { "claim": "rc-age-events", "statement": "Among children, responses at the peaks of pain events P01, P04 and P08 and ToM events T01 and T02 increase with age, controlling for motion, and no other event does at the corrected level.", "source": "Results, Reverse correlation analysis, paragraph 3; Figure 4a; Supplementary Figure 5", "quote": "Among children, three pain events (P01, P04, P08) and two ToM events (T01, T02) evoked significantly greater responses with age", "statistics": [ { "about": "P01, P04, P08", "test": "spearman partial correlation tests, motion as covariate", "statistic": "rss > .29", "p": 0.002, "p_relation": "<", "corrected": "Bonferroni, alpha .0026 for 19 events", "n": "122 children" }, { "about": "T01, T02", "test": "spearman partial correlation tests, motion as covariate", "statistic": "rss > .28", "p": 0.0026, "p_relation": "<", "corrected": "Bonferroni, alpha .0026 for 19 events", "n": "122 children" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Magnitudes at the 19 Supplementary Table 3 peak TRs. Under O7a (either O6 span), T02 (rs = .325), T07 (.352), P04 (.324) and T03 (.274) pass p < .0026; of the paper's five, P01 (rs = .193, p = .038), T01 (.270, p = .0036) and P08 (.019, p = .84) do not. Under O7b no event passes. 1 to 8 children per event are left out for an artifact at the peak TR.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "rc-t04-tom", "statement": "The ToM-network response at the peak of T04 rises with ToM score, controlling for age and motion.", "source": "Results, Reverse correlation analysis, paragraph 3; Figure 1b", "quote": "Responses in ToM regions during a third ToM event (T04) were significantly positively correlated with ToM score, controlling for age and motion", "statistics": [ { "test": "linear regression of T04 response magnitude on ToM score, age and motion", "statistic": "b = .4, t = 2.98", "p": 0.0035, "corrected": "alpha .007 for 7 ToM events", "n": "122 children" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "T04 magnitude on ToM score, age and motion (n = 114), t = 1.18, p = .24 (O7a, z over TRs 11 to 168); 1.14 and .71 under the other readings. No ToM event reaches p < .007 under any reading (smallest p = .061).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "rc-fb-no-difference", "statement": "Responses at the seven ToM event peaks do not differ between false-belief passers and failers.", "source": "Results, Reverse correlation analysis, paragraph 3; Figure 3c; Methods, fMRI data analysis, paragraph 11", "quote": "Response magnitude during ToM events did not differ significantly between children who pass and fail explicit falsebelief tasks", "statistics": [ { "test": "linear regressions of response magnitude on FB-Group, age and motion", "p": 0.08, "p_relation": ">", "n": "Pass 30, Fail 15" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Passers against failers, with age and motion, per ToM event. Under O7a no event differs (smallest p = .091, T05, with O6a; .072 with O6b); T03 permutation p = .72 (5000 permutations of the group labels). Under O7b T05 has p = .0497.", "deviations": [ "Assumed O7a, magnitudes from the z-normalized network timecourse, because the events and their peak timepoints are defined on that timecourse (Methods paragraphs 9 and 11, \"the timepoint with the highest average signal value in adults\"). Under O7b the claim would not be reproduced.", "The T03 permutation test permutes the group labels and refits the regression (5000 permutations, fixed seed); the paper cites R's lmPerm without its settings." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "rc-3yo-events", "statement": "Reverse correlation on the 3-year-olds alone finds 4 of the 12 adult pain events and 1 of the 7 adult ToM events; of its 32 s of pain events 14 s fall in adult pain events and 8 s in adult ToM event T04, and of its 8 s of ToM events 4 s fall in an adult ToM event and 4 s in adult pain event P01.", "source": "Results, Reverse correlation analysis, paragraphs 4 and 5; Figure 4a (dark borders)", "quote": "identified 4 of the 12 pain events and 1 of the 7 ToM events discovered in the ... adult sample. These events correspond to a subset of the timepoints that were identified as ToM or pain events in 3 year olds (Pain: 14/32 s, ToM: 4/8 s)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Reverse correlation on the 17 three-year-olds with the rc-events reading (O5a) and O6b. 3yo pain events 40 s overlapping 6 of 12 adult pain events (22 s) and T04 (6 s); 3yo ToM events 24 s overlapping 1 of 7 adult ToM events (4 s) and P01 (4 s). Both cross-network overlaps hold; times and the pain count do not. Under O6a, pain 40 s, 6 of 12, T04 6 s; ToM 20 s, 1 of 7, P01 0 s, so the P01 overlap fails. Adult events are those of Supplementary Table 3.", "deviations": [ "The rc-events reading ties between O6a and O6b. Assumed O6b, z-normalization over all 168 TRs, because Methods paragraph 9 z-normalizes each ROI timecourse first and leaves out the first 10 timepoints only from the t-tests. Under O6a the claim would not be reproduced.", "The criterion asks for the reading that reproduced rc-events, and no reading did. The reading that matched rc-events best (O5a) is used, an extension of the criterion." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "maturity-across", "statement": "Among children, the maturity (correlation with the adult mean) of each network's timecourse is predicted by stronger across-network anti-correlation, not by within-network correlation, with age and motion in the model.", "source": "Results, Relating functional maturity to inter-region correlations, paragraph 1; Figure 5", "quote": "We found that the maturity of the movie-driven timecourse in both ToM and Pain networks was predicted by the anti-correlation of regions across networks", "statistics": [ { "about": "ToM maturity, across-network effect", "test": "linear regression on across-network, within-network correlation, age and motion", "statistic": "b = -.4, t = -5.5", "p": 2.2e-07, "n": "122 children" }, { "about": "ToM maturity, within-ToM effect", "test": "same regression", "statistic": "b = .1, t = 1.5", "p": 0.14 }, { "about": "Pain maturity, across-network effect", "test": "linear regression on across-network, within-network correlation, age and motion", "statistic": "b = -.51, t = -7.4", "p": 2.8e-11, "n": "122 children" }, { "about": "Pain maturity, within-Pain effect", "test": "same regression", "statistic": "b = .13, t = 1.9", "p": 0.06 } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Maturity on across, within, age and motion (n = 122). Across effect ToM t = -4.52, p = 1.5e-05; Pain t = -3.55, p = .00055; both negative and significant but out of tolerance under every reading. Within-ToM t = 1.53, p = .13, not significant, as in the paper (under O2b with O3b it is significant, t = 1.99, p = .049); within-Pain t = 2.40, p = .018 (.0054 to .018 under every reading), significant, where the paper has p = .06. The criterion requires the within-network effects to be not significant, so the claim is not reproduced. z-scaled b across -.34 and -.28.", "deviations": [], "assumed": false, "disagreed": true, "data": "dataset", "where": null, "proposed": null }, { "claim": "maturity-within", "statement": "Among children, within-network correlations are associated with age but not with timecourse maturity.", "source": "Results, Relating functional maturity to inter-region correlations, paragraph 2", "quote": "Greater within-network correlations were strongly associated with age, but not with timecourse maturity", "statistics": [ { "about": "within-ToM, maturity effect", "test": "linear regression of within-network correlation on maturity, age and motion", "statistic": "b = .05, t = .48", "p": 0.63 }, { "about": "within-ToM, age effect", "test": "same regression", "statistic": "b = .3, t = 2.5", "p": 0.01 }, { "about": "within-Pain, maturity effect", "test": "linear regression of within-network correlation on maturity, age and motion", "statistic": "b = .07, t = .61", "p": 0.55 }, { "about": "within-Pain, age effect", "test": "same regression", "statistic": "b = .2, t = 1.96", "p": 0.05 } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Within-network on maturity, age and motion. Main reading, maturity t = .83, p = .41 (ToM) and t = 1.80, p = .075 (Pain); age t = 2.39, p = .019 and t = 4.22, p = 4.9e-05. Pain maturity p is .083 (O3b), .056 (O2b) and .040 (O2b, O3b).", "deviations": [ "Assumed O2a and O3a, as for irc-tom-behavior-age. Under O2b with O3b the Pain maturity effect is significant and the claim would not be reproduced." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "maturity-tom-behavior", "statement": "ToM timecourse maturity correlates with ToM score controlling for motion, but not once age is added.", "source": "Results, Relating functional maturity to inter-region correlations, paragraph 2", "quote": "while having an adult-like ToM timecourse was positively correlated with ToM behavior ... this relationship did not remain significant in a regression including age as an additional predictor", "statistics": [ { "about": "maturity against ToM score", "test": "spearman partial correlation test, motion as covariate", "statistic": "rs(119) = .54", "p": 1.3e-10, "n": "122 children" }, { "about": "ToM score effect with age", "test": "linear regression of ToM maturity on age, ToM score and motion", "statistic": "b = .1, t = 1.2", "p": 0.3 }, { "about": "age effect", "test": "same regression", "statistic": "b = .5, t = 4.2", "p": 5e-05, "p_relation": "<" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Partial Spearman of ToM maturity with ToM score, motion as covariate, .498, p = 6.1e-09. With age in the regression, ToM score t = .04, p = .97; age t = 4.46, p = 1.8e-05.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "maturity-fb", "statement": "ToM timecourse maturity does not differ between false-belief passers and failers.", "source": "Results, Relating functional maturity to inter-region correlations, paragraph 2", "quote": "Functional maturity of the ToM timecourse did not differ based on explicit false-belief task performance", "statistics": [ { "test": "linear regression, effect of FB-Group (pass vs. fail) with age and motion", "statistic": "b = -.12, t = -.35", "p": 0.73, "n": "Pass 30, Fail 15" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Group effect with age and motion t = 1.53, p = .13 (failers .095, passers .168), not significant.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "face-rc", "statement": "In bilateral fusiform face regions, the children's mean timecourse correlates with the adults' (r .86; 3-year-olds .72), reverse correlation in adults finds 7 face events (88 s) and in 3-year-olds 4 events (18 s), and face-event responses do not change with age.", "source": "Supplementary Figure 4; Results, Reverse correlation analysis, paragraph 1", "quote": "The average timecourse in bilateral fusiform ROIs in children was highly correlated with that of adults (pearson correlation: r=.86, p<1.0x10-47)", "statistics": [ { "about": "children against adults", "test": "pearson correlation", "statistic": "r = .86", "p": 1e-47, "p_relation": "<" }, { "about": "3-year-olds against adults", "test": "pearson correlation", "statistic": "r = .72", "p": 1e-26, "p_relation": "<" }, { "about": "age effect on face-event magnitudes", "test": "spearman partial correlation including motion as covariate", "statistic": "|r|s < .24", "p": 0.01, "p_relation": ">", "corrected": "Bonferroni, alpha .0071 for 7 events" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Children against adults r = .818 (p = 2.8e-39), in tolerance; 3yo r = .589 (p = 3.8e-16), out of tolerance and above the paper's bound p < 1e-26. Under the rc-events reading (O5a, O6b) 10 adult face events (108 s) and 6 3yo events (32 s); no reading gives 7 adult events (8 to 10, 72 to 108 s). No face-event magnitude changes with age at .0071 (smallest p = .010). The correlations do not both hold, so neither the reproduced nor the partial outcome of the criterion applies.", "deviations": [ "The fusiform ROIs are the deposited rFFA (1019 voxels) and lFFA (531), the sizes of Supplementary Table 2.", "Face-event peaks are the found adult events' peaks, as the paper gives no face peak TRs." ], "assumed": false, "disagreed": false, "data": "with_inputs", "where": null, "proposed": null }, { "claim": "face-overlap", "statement": "The overlap of face events with ToM events (4 TRs) and with pain events (15 TRs) does not differ from chance.", "source": "Supplementary Note 1", "quote": "Thus, the amount of overlap between face and ToM events, and face and pain events, did not differ from that expected by chance", "statistics": [ { "about": "face and ToM", "test": "permutation test, 1000 permutations", "statistic": "111/1000", "p": 0.11 }, { "about": "face and pain", "test": "permutation test, 1000 permutations", "statistic": "928/1000", "p": 0.93 } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_richardson_2018.ipynb", "notes": "Face events overlap ToM events by 7 TRs (p = .052 for fewer than chance) and pain events by 25 TRs (p = .983) under the rc-events reading; .098 to .166 and .94 to .96 under the others. Neither is below .05. The overlaps differ from the paper's 4 and 15 TRs.", "deviations": [], "assumed": false, "disagreed": false, "data": "with_inputs", "where": null, "proposed": null }, { "claim": "replication-sample", "statement": "In an independent sample of 20 adults, reverse correlation finds 7 ToM and 9 pain events, all of them also found in the main adult sample; P08, P09 and P11 are not replicated.", "source": "Supplementary Figure 6; Methods, fMRI data analysis, paragraph 3", "quote": "Seven ToM and nine pain events were identified in the reverse correlation analysis of this independent sample of adults", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "other", "notebook": null, "notes": "The independent sample of 20 adults is not in the snapshot, and neither the paper nor the dataset's README names an archive that holds it. The claims file marks the claim `elsewhere` with no archive; as the data are public nowhere, data_missing would apply, which the checker does not allow for an `elsewhere` claim. The dataset's README lists the events that replicated across the two adult samples; they overlap all seven ToM events and the pain events of Supplementary Table 3 except P08, P09 and P11, as the claim states. This restates the authors' result and is not the second sample's data, so the claim stays blocked.", "deviations": [], "assumed": false, "disagreed": false, "data": "elsewhere", "where": "Not deposited. The paper names no archive for the 20 adults scanned by Evelina Fedorenko and colleagues.", "proposed": null }, { "claim": "hand-coded-events", "statement": "Of the adult reverse-correlation events, 6 of 7 ToM events but only 2 of 12 pain events fall at least partly within the mental, social or pain events hand-coded in an earlier study.", "source": "Supplementary Figure 7", "quote": "most ToM events identified by the reverse correlation analysis were at least partially included in the original coding (6/7 ToM events labeled as Mental or Social), only two of twelve pain events were included in the coding", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "other", "notebook": null, "notes": "The hand-coded events of Jacoby et al. 2016 are not in the snapshot (its README lists only reverse-correlation events) or the paper. The Saxe lab web page (https://saxelab.mit.edu/theory-mind-and-pain-matrix-localizer-movie-viewing-experiment) offers the experiment script movloc.zip (version of 29 March 2019) whose mov_localizer.m holds them, but with no DOI or repository commit, so it cannot be pinned as the protocol requires for an outside input.", "deviations": [], "assumed": false, "disagreed": false, "data": "with_inputs", "where": null, "proposed": null } ], "findings": [ { "kind": "other", "statement": "With the paper's own deposited preprocessed images, artifact and CompCor regressors and ROI masks, and the pipeline of the Methods, the inter-region correlations differ from the paper's, higher within networks and less negative across them (adults .604, .398 and -.051 against .48, .35 and -.17). In the 3-year-olds within-ToM is higher (.326 against .21) but within-Pain is lower (.179 against .23; non-bilateral Pain .146 against .17), so the offset is not the same in every group. Several downstream results (within-ToM difference between false-belief passers and failers, age effects at event peaks, the T04 effect of ToM score) do not come out as reported. Either the deposited data or the described pipeline differs from what produced the paper's numbers.", "found": null, "status": null }, { "kind": "methods_unavailable", "statement": "For the authors, which timepoints entered the inter-region correlations (all 168 TRs, or TRs 11 to 168 as in the reverse correlation), and whether the artifact timepoints were regressed with one indicator each? irc-tom-behavior-age, irc-fb-across and maturity-within change outcome with this choice; this attempt assumed all 168 TRs and one indicator each, as in the deposited regressor files.", "found": null, "status": null }, { "kind": "methods_unavailable", "statement": "For the authors, over which timepoints was each ROI timecourse z-normalized before the reverse correlation, and were child event magnitudes taken from the z-normalized timecourse? rc-3yo-events depends on the first (all 168 TRs assumed) and rc-fb-no-difference on the second (z-normalized assumed). The alpha of the reverse-correlation t-tests is also not stated; p < .05 uncorrected matches Supplementary Table 3 best.", "found": null, "status": null }, { "kind": "methods_unavailable", "statement": "The analysis code is available only on request, and the high-pass filter implementation (SPM discrete cosine assumed) is not stated.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "participants.tsv lists 2 seven-year-olds and 21 8-12-year-olds with 3.13 mm voxels and no slice gap, where the Methods say 3 and 20. The voxel-size column name is misspelled (\"Voxel slize\"), and the no-gap level is stored as an empty value where participants.json defines the level \"None\".", "found": null, "status": null }, { "kind": "other", "statement": "19 participants (14 aged 8-12, 2 adults, one each 4, 5 and 7 years) have face-ROI voxels without data at some timepoint, where Supplementary Fig. 4 counts 17 with incomplete ventral coverage (12 aged 8-12, 2 adults, one each 3, 4 and 7 years); the minimum of 101 voxels with data agrees with its \"at least 100 voxels\". The coverage criterion may differ.", "found": null, "status": null }, { "kind": "other", "statement": "The independent sample of 20 adults that defined the ToM and pain ROIs and replicated the events is not deposited anywhere; the ROIs it produced are in the dataset.", "found": null, "status": null } ], "reconciliation": null, "verifications": [ { "date": "2026-10-28", "kind": "agent", "by": "Independent agent run by CatalystNeuro (verifier-1, same harness, separate context, did not write the notebook)", "reviewed": "Round 1. The executed notebook with the conclusion cells removed, the helper module and the claims file, with the paper. The summary cell was not removed by the handover, as it shared a cell with the blocked claims, and the verifier read it after forming most of its outcomes. It queried the archive's file list for the blocked claims and tested the helper functions on synthetic inputs. Of 42 claims it agreed with 39 outcomes and disagreed on irc-williams, rc-child-adult-timecourse and maturity-across.", "agreed": 39, "of": 42, "disagreements": [ { "claim": "irc-williams", "outcome": "not_reproduced", "reason": "Under both Williams readings, within-Pain vs Scene (p .010) and within-Pain vs M1 (p .004) come out significant where the paper has n.s. That is 8/10 comparisons as in the paper, and the criterion has no partial clause." }, { "claim": "rc-child-adult-timecourse", "outcome": "not_reproduced", "reason": "ToM 3yo r .212 (p .0076) and Pain 3yo .531 are out of tolerance. The 3yo ToM p misses the paper's \"all p < .0005\" bound, and Pain 3yo/4yo miss its \"p < 1e-13\" bound. All are significant at the Bonferroni \u03b1 .01, so a laxer reading of the criterion would give partial. The notebook checks every bin only against p < .0005." }, { "claim": "maturity-across", "outcome": "not_reproduced", "reason": "The across-network effects hold (t -4.5 and -3.6, out of tolerance). The within-Pain effect is significant under every reading (p .005 to .018), which fails the criterion's condition." } ] }, { "date": "2026-10-28", "kind": "agent", "by": "Independent agent run by CatalystNeuro (verifier-2, same harness, separate context, did not write the notebook)", "reviewed": "Round 2. The full executed notebook after the round-1 revision, the helper module, the claims file and the attempt record, with the paper. Of 42 claims it agreed with 41 outcomes and disagreed on rc-overlap. It found no misreading in the claims file, so no question was added. It also questioned the README's run time, which now gives the times `tools/entry.py all` reported.", "agreed": 41, "of": 42, "disagreements": [ { "claim": "rc-overlap", "outcome": "not_reproduced", "reason": "Disagrees. The criterion defines only a reproduced outcome (\"one overlapping timepoint is found and p <= .01 under a reading\"), does not invoke Rule S and has no partial clause. The overlap is 0 against 1 with tolerance 0 (cell 86). By the record's reasoning for irc-williams it is not_reproduced. Partial could only be defended by applying Rule S's partial clause, which this criterion does not name; the record should state that reasoning, or change the outcome. In round 1 the verifier agreed and missed this." } ] } ] } ], "entries": [ { "name": "openneuro-ds000228-richardson-2018", "type": "reproduction", "title": "Development of the social brain from age three to twelve (Richardson et al. 2018)", "description": "Tests the findings of Richardson, Lisandrelli, Riobueno-Naylor and Saxe (2018) on OpenNeuro ds000228, which holds movie-viewing fMRI of 122 children aged 3 to 12 and 33 adults, with the paper's preprocessed images, artifact and CompCor regressors, group ROIs and behavioural scores. It covers the behavioural results, motion, inter-region correlations within and across the ToM and pain networks, reverse correlation of network timecourses, functional maturity, and the face-region analyses.", "datasets": [ "openneuro:ds000228" ], "papers": [ "10.1038/s41467-018-03399-2" ], "derived_from": [], "notebooks": [ { "path": "reproduce_richardson_2018.ipynb", "title": "Reproducing Richardson et al. (2018) from OpenNeuro ds000228", "executed": "notebooks/openneuro-ds000228-richardson-2018/reproduce_richardson_2018.html" } ] } ] }