{ "dataset": { "archive": "dandi", "id": "000575", "label": "DANDI:000575" }, "page": "datasets/dandi/000575/", "generated": "2026-11-05T07:41:23+00:00", "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.", "outcome_counts": { "reproduced": 8, "partially_reproduced": 3, "not_reproduced": 6, "blocked": 0 }, "outcome_counts_with_outside_data": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_elsewhere": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_links": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "attempts": [ { "id": "000575-boran-2022-01", "dataset": { "archive": "dandi", "id": "000575", "version": "0.231010.1811", "doi": "10.48324/dandi.000575/0.231010.1811" }, "key": "dandi:000575", "label": "DANDI:000575", "dataset_version": "0.231010.1811", "paper": "10.1016/j.neuroimage.2022.119123", "paper_title": "Persistent neuronal firing in the medial temporal lobe supports performance and workload of visual working memory in humans", "date": "2026-10-30", "kind": "agent", "accountable": "CatalystNeuro", "agent": { "harness": "Claude Code", "model": "claude-opus-5-5", "protocol": "pilot-12" }, "runner": { "platform": "GitHub Actions", "url": "https://github.com/catalystneuro/encore/actions/runs/41115883209" }, "tier": "agent_verified", "deprecated": "", "deprecated_on": null, "entry": "000575-boran-2022", "entry_title": "Persistent firing of human MTL neurons in visual working memory (Boran et al. 2022)", "description": "Tests the findings of Boran, Hilfiker, Stieglitz, Sarnthein and Klaver (2022) on dandiset 000575, which holds the trials, answers and sorted units of 16 sessions from 13 epilepsy patients performing a change detection task with 1 to 6 colored squares. It checks the behavior, the neuron counts and firing rates, the maintenance and test neurons, the demixed PCA of the population and the decoding of workload and performance.", "notebooks": [ { "path": "reproduce_boran_2022.ipynb", "title": "Reproducing Boran et al. (2022) from dandiset 000575", "executed": "notebooks/000575-boran-2022/reproduce_boran_2022.html" } ], "outcomes": [ { "claim": "behavior-accuracy-by-set-size", "statement": "Performance was almost perfect at set sizes 1 (98% correct) and 2 (99%) and fell at 4 (88%) and 6 (73%), a significant effect of set size in a permuted repeated-measures ANOVA over 16 sessions; average performance was 89%.", "source": "Results, 3.1 Task and behavior, paragraph 1; Figure 1b; Discussion 4.1, paragraph 2", "quote": "The partici- pants showed almost perfect performance at set size of 1 (98% correct responses) and 2 (99 %), and a decrease at 4 (88%) and 6 (73%) ( Fig. 1 b, permuted repeated-measures ANOVA, F 3,45 = 89.79, p = .0002, 16 ses- sions)", "statistics": [ { "test": "permuted repeated-measures ANOVA", "statistic": "F(3, 45) = 89.79", "p": 0.0002, "n": "16 sessions" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Trials pooled over sessions (reading a) give 97.76, 97.90, 84.32 and 72.60% correct at set sizes 1, 2, 4 and 6 and 88.15% overall; the mean of session percentages (reading b) gives 97.73, 97.90, 84.36, 72.54 and 88.15%. The repeated-measures ANOVA over 16 sessions gives F(3, 45) = 89.78 (paper 89.79), permutation p = 0.0002 with set-size labels shuffled within sessions (4999 permutations, largest null F 10.81); permuting the four session values instead gives the same p.", "deviations": [ "The sessions hold 3038 trials (174 to 192 per session), where the paper states 192 per session; the 2678 correct trials equal the paper's count." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "behavior-capacity", "statement": "Across sessions the median working memory capacity was 3.2 by Pashler's K (range 2.2 to 5) and 3 by Cowan's K (range 1.9 to 4.5), so participants held at least 3 squares and set sizes 4 and 6 exceeded capacity.", "source": "Results, 3.1 Task and behavior, paragraph 1; Figure 1b; Table 1", "quote": "Across all sessions, the median memory capacity according to Pashler ( K P ) was 3.2 (range 2.2\u20135). According to Cowan's K ( K C ) me- dian memory capacity was 3 (range 1.9\u20134.5)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Per participant (reading a), Pashler's Kmax has median 3.23 and range 2.25 to 5.00, and Cowan's K as its maximum over set sizes (reading i) median 3.00 and range 2.11 to 4.50, all within tolerance. Per session (reading b) the medians are 3.46 and 3.00 and the ranges 2.25 to 5.00 and 1.92 to 4.50, also within tolerance. Cowan's K at set size 6 (reading ii) has median 2.67 per participant (just outside 3 +- 0.3) and 2.75 per session.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "table1-capacity-and-neurons", "statement": "Table 1 gives, per participant, the number of sessions, the memory capacity Kmax (2.41, 2.25, 3.04, 3.00, 2.87, 3.88, 3.00, 3.23, 3.58, 3.81, 5.00, 3.83, 4.50 for participants 1 to 13) and the number of neurons (33, 36, 106, 85, 29, 96, 48, 150, 23, 62, 11, 57, 73).", "source": "Table 1", "quote": "", "statistics": [], "kind": "finding", "inferred": "Numbers read from Table 1; no sentence states them. For participants 3, 8 and 12 Kmax is the mean of two sessions (2.50 and 3.58, 3.33 and 3.13, 4.00 and 3.67).", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "All 13 neuron counts and all 13 Kmax values equal Table 1 to two decimals, including the session values of participants 3 (2.50, 3.58), 8 (3.33, 3.13) and 12 (4.00, 3.67).", "deviations": [ "Participant 12's sessions are each deposited twice; each is counted once, by the session number in the file identifier." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "behavior-rt-set-size", "statement": "Mean response time on correct trials (2678 trials) increased with set size, by 118 ms per item, a significant effect in a permuted repeated-measures ANOVA over 16 sessions.", "source": "Results, 3.1 Task and behavior, paragraph 1; Figure 1c", "quote": "The mean response time (RT) for the correct trials (2678 trials) increased with set size ( Fig. 1 c, 118 ms/item, permuted repeated-measures ANOVA, F 3,45 = 48.47; p = .0002, 16 sessions)", "statistics": [ { "test": "permuted repeated-measures ANOVA", "statistic": "F(3, 45) = 48.47", "p": 0.0002, "n": "16 sessions, 2678 correct trials" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Measured values are those of reading (b), per-session median RT, with slope reading (i); no reading matches every value. There are 2678 correct trials. RT rises with set size in all 16 sessions. On per-session median RT the ANOVA gives F(3, 45) = 48.47, the paper's value, with permutation p = 0.0002; on mean RT (reading a) F = 61.36, outside tolerance. The slope is 75.8 (i), 76.2 (ii) and 75.8 (iii) ms/item on mean RT and 68.8, 76.2 and 68.8 on median RT, all outside 118 +- 11.8. The effect and its test hold, the slope is about 60% of the reported one. The text says the ANOVA is on mean RT, and its F equals the one on median RT here (Figure 1c shows medians). The criterion has no partial clause; partial is the author's judgment, as the rise, its test and F hold and the slope does not. As a variant outside the claims file, the fit of the median RTs on the condition step (1 to 4) gives 118.1 ms per step, the paper's value (mean RTs 129.9); whether the claims file should name this reading is asked in questions/000575.yaml, and the outcome is judged on the readings the file names.", "deviations": [ "RT is the stored response timestamp minus trial start minus 1.7 s, read as the time from test onset (data check; the conversion adds 1.7 s to the stored response time). The slopes and F values do not depend on this offset." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "neurons-recorded", "statement": "809 neurons were recorded, 454 in hippocampus, 209 in entorhinal cortex and 146 in amygdala, 50 plus or minus 25 (range 11 to 96) per session.", "source": "Results, 3.2 Neuronal firing, paragraph 1", "quote": "We recorded the ac- tivity of 809 neurons from the hippocampus (Hipp, n = 454), entorhi- nal cortex (Ent, n = 209), and amygdala (Amg, n = 146) across all microelectrodes. We recorded 50 \u00b1 25 (range 11\u201396) neurons per ses- sion.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "809 neurons, 50.6 +- 25.4 per session (range 11 to 96), all within tolerance. By tip location (reading a), with the 133 units whose tip contact has no MTL label placed by the nearest contact of their bundle that has one (a2), 467 are in hippocampus, 196 in entorhinal cortex and 146 in amygdala, all within tolerance of 454, 209 and 146. Placing those units by their electrode name (a1, the rule of round 1) gives 495, 168 and 146, the entorhinal count outside tolerance. By electrode name (reading b) the counts are 361, 217 and 226, with 5 units on bundles the reading does not cover (PCL in participant 8, HL and HR in participant 11); only the entorhinal count is within tolerance. The outcome rests on (a2). What changed in round 1: the outcome was partially reproduced under (a1); the verifier found that no other placement of the 133 units had been tried, and (a2) was examined in reply. It moves the 28 units on PHR of participant 8, whose contacts 2 and 3 are labelled PhG, to entorhinal cortex; the other 105 units keep their region under both rules. The choice was made after seeing the counts it gives, which the verifier should weigh.", "deviations": [ "Reading (a) takes the location of the microwire tips from the atlas labels of each bundle's contacts, as the microwires protrude 4 mm from the electrode tip (Methods 2.3) and Figure 1d shows the \"Location of the microelectrodes at the tip of the depth electrodes\" color-coded by region. The files hold no location for the wires themselves. PhG (parahippocampal gyrus) is read as entorhinal cortex. The tip contact (contact 1) gives the region where it has an MTL label. Assumption (a2), for the 133 units whose tip contact is labelled unspecific or FuG (all on hippocampal electrodes), the label of the contact nearest the tip with an MTL label, and the electrode name where no contact has one (60 units). It is preferred to the electrode name (a1) because it uses the bundle's own atlas labels before its target name; both are shown, and Figure 1d colours every site as one of the three regions. The grounding is weak: (a2) differs from (a1) only for the 28 units on PHR of participant 8, and it places them by contacts 2 and 3, which lie further from the protruding wire tips than contact 1. Under (a1) the entorhinal count is 168, below the tolerance floor of 188.1." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "spike-sorting-quality", "statement": "Over all neurons, 0.79 plus or minus 1.54% of inter-spike intervals were shorter than 3 ms, most neurons were below 0.5%, and the peak signal-to-noise ratio of the mean waveform was 3.81 plus or minus 0.82.", "source": "Results, 3.2 Neuronal firing, paragraph 1", "quote": "Over all neu- rons, the percentage of inter-spike intervals (ISI) shorter than 3 ms was 0.79 \u00b1 1.54%. The majority of neurons had short ISI percentages less than 0.5%. ... SNR waveform peak over all neurons was 3.81 \u00b1 0.82.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Over within-trial intervals 0.79 +- 1.77% of ISIs are under 3 ms (median 0.31%, maximum 26.46%); the mean equals the paper's, the SD is outside 1.54 +- 0.154. 506 of 809 units (62.5%) are below 0.5%. The SNR, peak of the mean waveform over the mean waveform SD (reading a), is 5.93 +- 2.31; with the SD at the peak sample 5.96 +- 1.43, with the SD of the first five samples 5.92 +- 2.78, all against 3.81 +- 0.82. The criterion's partial condition needs both ISI values, so neither condition is met.", "deviations": [ "Reading (i) of the ISI, the whole recording, cannot be computed, as the files hold spikes only inside trial windows. Reading (b) of the SNR, a noise level stored by the spike sorter, cannot be computed, as the Units table has none." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "firing-rates-by-period", "statement": "The median (IQR) firing rate over all neurons was 1.09 (0.52 to 2.43) Hz over the trial, 1.12 (0.54 to 2.56) Hz during fixation, 1.10 (0.53 to 2.50) Hz during encoding, 1.09 (0.54 to 2.51) Hz during maintenance and 1.02 (0.49 to 2.45) Hz during test.", "source": "Results, 3.2 Neuronal firing, paragraph 1", "quote": "We identified the firing rate over all neurons as median (inter-quartile range, IQR) 1.09 (0.52\u20132.43) Hz throughout the trial du- ration. Median (IQR) firing rate over all neurons was 1.12 (0.54\u20132.56) Hz during fixation, 1.10 (0.53\u20132.50) Hz during encoding, 1.09 (0.54\u2013 2.51) Hz for maintenance and 1.02 (0.49\u20132.45) Hz during test.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Correct trials (reading a), trial from fixation onset to test offset (reading i); every median and quartile is within 10% of the paper's. All trials (reading b) also match every value (medians 1.088, 1.117, 1.086, 1.099 and 1.044 Hz).", "deviations": [ "Reading (ii) of the trial, from the warning signal to the end of the inter-trial interval, cannot be computed, as no spikes are stored between trials." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2c-maintenance-neurons", "statement": "Maintenance neurons, firing more during maintenance than during fixation, were found in all three regions, 55 of 454 in hippocampus, 21 of 209 in entorhinal cortex and 16 of 146 in amygdala, each a significant proportion.", "source": "Results, 3.2 Neuronal firing, paragraph 4; Figure 2c", "quote": "These maintenance neurons were defined by their higher firing rates during the maintenance period than during the fixation period ... Maintenance neurons were found in the hippocampus, the amygdala and the entorhinal cortex (per- mutation t-test, p < .05, Fig. 2 b, c).", "statistics": [ { "test": "permutation test against scrambled labels", "p": 0.002, "n": "each region, 454, 209 and 146 neurons" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Measured values are the paired test (reading a) at 5000 permutations per neuron, the check the claims file names, with regions by tip location (a2). Maintenance neurons are found in all three regions under both tests and every region reading, each count significant against 499 within-trial shuffles of the period labels (p = 0.002 for every region and reading); 98 in all with the paired test at 1000 permutations, 94 at 5000, and 87 with the unpaired test, against 92. At 5000 permutations the paired counts are 58, 21 and 15 (a2) and 59, 20 and 15 (a1), all within tolerance of 55, 21 and 16. At 1000 permutations the counts move with the draw: over 10 draws (a2) gives 55 to 59, 20 to 22 and 14 to 16, and the draw used for the region test gives 61, 21 and 16, the hippocampal count 0.5 above tolerance. The unpaired test gives 54, 20 and 13 (a2), the amygdala count outside. By electrode name the counts are 47 to 48, 28 and 19 to 22, none within tolerance. What changed in round 1: the outcome was partially reproduced on the 1000-permutation draw (62, 20, 16 by tip location, a1). The 5000-permutation counts were added at the verifier's request and are within tolerance under both tip-location rules; the outcome takes them, as the more exact p values, and rests on the tip-location reading of the regions (neurons-recorded).", "deviations": [ "The p value of each per-neuron test is (number of permutation t values at least as large in absolute value + 1) / (number of permutations + 1), with 1000 permutations for the region tests and 5000 for the counts, the claims file's standard choice and its check." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2d-workload-modulated-maintenance-neurons", "statement": "Among maintenance neurons, a significant number increased firing with workload in hippocampus (10 of 55, 18%) and entorhinal cortex (5 of 21, 24%), and a significant number decreased firing with workload in hippocampus (6 of 55, 11%); in entorhinal cortex 2 decreased and in amygdala 1 increased, not significant.", "source": "Results, 3.2 Neuronal firing, paragraph 5; Figure 2d", "quote": "we found a significant number of maintenance neurons that showed an increase in firing with workload in the hippocampus (10/55 = 18%, permutation test against scrambled labels, p = .002, Fig. 2 d) and entorhinal cortex (5/21 = 24%, permutation test against scrambled labels, p = .002). We also found a significant number of maintenance neurons that showed a decrease in firing rate with workload in the hippocampus (6/55 = 11%, permutation test against scrambled labels, p = .002).", "statistics": [ { "about": "increase with workload, hippocampus", "test": "permutation test against scrambled labels", "p": 0.002, "n": "55 hippocampal maintenance neurons" }, { "about": "increase with workload, entorhinal cortex", "test": "permutation test against scrambled labels", "p": 0.002, "n": "21 entorhinal maintenance neurons" }, { "about": "decrease with workload, hippocampus", "test": "permutation test against scrambled labels", "p": 0.002, "n": "55 hippocampal maintenance neurons" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Measured values are for the paired maintenance neurons, the per-neuron permutation t-test (reading a) and regions by tip location (a2; a1 gives the same counts). Hippocampal increase 2 (p = 0.43 against 499 shuffles of set-size labels), entorhinal increase 0 (p = 1), hippocampal decrease 2 (p = 0.49), entorhinal decrease 2 (p = 0.09), amygdala increase 1 (p = 0.29). By electrode name the counts are 2 (p = 0.30), 1 (p = 0.50), 2 (p = 0.38), 3 (p = 0.042) and 0. The unpaired maintenance neurons give 2, 0, 2, 2 and 1 by tip location and 2, 1, 2, 2 and 0 by electrode name, no named count below p = 0.25; the cluster-based test (reading b, 99 shuffles) 1, 0, 1, 0 and 0 under both region readings. With every session's spikes moved by three trials (variant, electrode name) the counts are 1, 1, 1, 1 and 0. The expected count under shuffled labels is 0 to 1. None of the three named counts is near 10, 5 or 6 or significant, under any region reading. The data check suggests the spikes may be misaligned with the trial labels, so this failure may come from the deposited data; moving every session by three trials does not recover the counts.", "deviations": [ "Reading (b), the cluster-based test, uses rates in 500 ms windows stepped by 50 ms (the Figure 2a legend gives 500 ms bins with a 1 ms step), a two-sided t threshold at p < .05 and the 95th percentile of the largest cluster mass over 1000 label permutations; its counts use 99 label shuffles, not 499, for time." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "maintenance-neurons-correct-vs-incorrect", "statement": "More hippocampal neurons were classified as maintenance neurons on correct trials than on incorrect trials (55 of 454, 12%, against 30 of 454, 7%).", "source": "Results, 3.2 Neuronal firing, paragraph 6", "quote": "There were more of these hippocam- pal neurons for correct trials vs incorrect trials (55/454 = 12% vs. 30/454 = 7%, permutation test against scrambled labels, p = .002).", "statistics": [ { "test": "permutation test against scrambled labels", "p": 0.002, "n": "454 hippocampal neurons" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Paired test, all trials of each kind (reading a), regions by tip location (a2): 56 hippocampal maintenance neurons on correct trials and 18 on incorrect ones; the correct count is within tolerance of 55, the incorrect count is not (30). The difference of 38 is not larger than with correct and incorrect labels shuffled within sessions (median 40, 95th percentile 50, p = 0.67); unpaired, 54 and 19 (p = 0.62). With the electrode-name fallback (a1), 57 and 19 (p = 0.71; unpaired 55 and 19, p = 0.66). By electrode name, 47 and 13 (p = 0.33), unpaired 42 and 13 (p = 0.50). With correct trials subsampled to the incorrect count (reading b) the correct count averages 15.7 (a2), 16.9 (a1) and 12.6 by electrode name, close to the incorrect counts. The correct-trial counts here differ by one or more from fig2c because the per-neuron permutations are drawn anew. The test compares labels across trials, which the data check suggests may be misaligned with the spikes.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig3b-test-neurons", "statement": "Test neurons, firing more after the test array than during encoding and maintenance, made up 46 of 809 neurons (6%), half the number of maintenance neurons, and were found in all regions (17 of 454 in hippocampus, 19 of 209 in entorhinal cortex, 10 of 146 in amygdala).", "source": "Results, 3.2 Neuronal firing, paragraph 7; Figure 3b", "quote": "These test neurons were defined based on increased firing only during the presentation of the test array relative to encoding and maintenance (permutation t-test, Fig. 3 a). With a rate of 46/809 = 6%, these were half the number of maintenance neurons ( Fig. 3 b).", "statistics": [ { "test": "permutation test against scrambled labels", "p": 0.002, "n": "each region, 454, 209 and 146 neurons" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Measured values are reading (a), paired, with regions by tip location (a2). Test neurons are found in all three regions under every reading and both region readings, each count significant against shuffled period labels (p = 0.002 to 0.046). The totals are 86 (reading a), 42 (reading b, test above encoding and above maintenance) and 89 (reading c, test against the two periods joined) paired, and 79, 36 and 84 unpaired, against 46. By tip location (a2), reading (a) paired gives 35, 34 and 17 by region; reading (b) paired 22, 11 and 9, the total and the amygdala count within tolerance; reading (b) unpaired 16, 9 and 11, hippocampus and amygdala within tolerance. With the electrode-name fallback (a1), reading (a) paired gives 38, 31 and 17 and reading (b) paired 23, 10 and 9. By electrode name, reading (a) paired gives 35, 40 and 11, and reading (b) paired 20, 18 and 4, the total and the entorhinal count within tolerance. No combination of readings puts all four counts within tolerance. Under reading (b) test neurons are 43% of the 98 maintenance neurons, about half as the statement says. The criterion has no partial clause; partial is the author's judgment, as test neurons are significant in every region and the reading (b) total matches.", "deviations": [ "The significance of reading (b) uses 99 within-trial shuffles of the three period labels with 200 permutations per neuron and test, for time." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4a-dpca-separates-set-sizes", "statement": "Demixed PCA of the population separated set sizes 1, 2, 4 and 6; dPCs 2, 3 and 4 explained 36% of the variance (significant against scrambled labels), the separation appeared early in the trial, and the first 15 dPCs explained 81.4% of the variance.", "source": "Results, 3.3 Neuronal population analysis, paragraph 1; Figure 4a", "quote": "dPCA clearly distinguished between set sizes 1, 2, 4, and 6. For better visualization we formed a three-dimensional space from three demixed principal components (dPCs 2, 3 and 4, explaining 36% of the variance, permutation test against scrambled labels, p = .002, Fig. 4 a). ... we included the first 15 dPCs that explain 81.4% of the signal variance", "statistics": [ { "test": "permutation test against scrambled labels", "p": 0.002, "n": "809 neurons" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "With no regularisation (reading c) and kernel (a), dPCs 2 to 4 explain 32.88% and the first 15 dPCs 81.41%, both within tolerance (kernel b, 30.97% and 77.74%; the cross-validated lambda of reading d, 22.21% and 48.20% for kernel a). dPCs 2 to 4 are time components. With set-size labels shuffled within sessions and dPCA refit (99 times), the set-size variance in the first 15 dPCs is 19.23% (median) against 19.50% observed, p = 0.35; dPCs 2 to 4 give p = 0.27 and the mean distance between set-size trajectories during maintenance p = 0.53. So the variance values reproduce, but the set sizes are not separated beyond chance, and the criterion's test fails. In all 99 shuffled fits dPCs 2 to 4 are time components too, so their variance cannot detect set-size information; the set-size variance of the first 15 dPCs is the test that can. With every session's spikes moved by three trials (variant) the values are 32.59% and 80.34%.", "deviations": [ "Spike counts are binned at 10 ms before smoothing, not 1 ms; with a 167 or 200 ms SD kernel this is a standard choice that changes the smoothed rates very little. The smoothing is renormalised at the trial edges, where no spikes are stored. Explained variance is computed as in the dPCA toolbox (Kobak et al. 2016), from the python package dPCA 1.0.5.", "The null uses 99 shuffles, not 499, for time; its smallest p is 0.01." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4b-speed-lowest-in-maintenance", "statement": "Multidimensional speed in the 15-dPC space was highest during encoding and lowest during maintenance, lower than in encoding and test.", "source": "Results, 3.3 Neuronal population analysis, paragraph 1; Figure 4b", "quote": "Multidimensional speed was highest during encoding, when maintenance neurons ramped up their firing. Speed was lowest during maintenance (permutation t-test against encoding and test periods, p = .0005).", "statistics": [ { "test": "permutation t-test against encoding and test periods", "p": 0.0005, "n": "500 dPCA runs on 20% subsamples" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Reading (a), 500 dPCA runs on 20% subsamples of neurons with replacement: mean speed 10.43 in fixation, 18.79 in encoding, 12.97 in maintenance and 16.47 in test; maintenance below encoding (t = -35.9) and test (t = -22.1), p = 0.001 each, the smallest possible with 1000 permutations; 98.2% of runs have maintenance below both. Reading (b), 100 runs subsampling trials, gives the same order (34.42, 51.35, 41.34, 46.88) and p values. Fixation is slower than maintenance, so the statement's \"lowest during maintenance\" does not hold against fixation; the criterion leaves that comparison out.", "deviations": [ "Reading (b) uses 100 runs, not 500, for time.", "dPCA as in fig4a with kernel (a) and no regularisation, the combination whose variance values match." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4c-distance-largest-in-maintenance", "statement": "The pairwise distance between set-size trajectories was larger during maintenance than during encoding, and the largest of all periods, consistent with attractors during maintenance.", "source": "Results, 3.3 Neuronal population analysis, paragraph 1; Figure 4c", "quote": "Analysing the pairwise distance between trajecto- ries that corresponded to different set sizes, we found that the distances during maintenance were larger than during encoding (permutation t- test, p = .0005, Fig. 4 c).", "statistics": [ { "test": "permutation t-test", "p": 0.0005, "n": "500 dPCA runs on 20% subsamples" } ], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "In the runs of fig4b (reading a) the mean distance between set-size trajectories is 5.777 in fixation, 5.013 in encoding, 5.375 in maintenance and 5.273 in test; maintenance exceeds encoding (t = 8.1, p = 0.001) and test (p = 0.011), but fixation is the largest. Reading (b) gives the same (30.66, 26.05, 27.67, 26.89; p = 0.001). The fig4a null shows that the maintenance distance of the full population does not exceed that with shuffled set-size labels (p = 0.53), so these distances do not show a separation of set sizes: the difference between maintenance and encoding reflects how the distance varies with the period, and the partial outcome, which the criterion gives when only its first condition holds, carries little weight.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4d-decode-workload-all", "statement": "During maintenance, workload (set sizes 1 and 2 against 4 and 6) was decoded from all neurons pooled with median accuracy 87.5%, above chance; leaving out any one participant gave accuracies with IQR 85.7 to 87.6%.", "source": "Results, 3.4 Decoding of information during the trial, paragraph 1; Figure 4d", "quote": "For the maintenance periods, we found that the workload (set size 1, 2 vs. set sizes 4, 6) of each trial could be decoded with median decoding accuracy of 87.5% when all neurons were pooled ( Fig. 4 d, permuta- tion test against scrambled labels, p = .001). In a leave-one-out anal- ysis, the exclusion of any participant resulted in decoding accuracy in the IQR [85.7\u201387.6%]", "statistics": [ { "test": "permutation test against scrambled labels", "p": 0.001, "corrected": "Bonferroni over 4 regions", "n": "809 neurons" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Measured values are pseudo-trials (c), z-scored features (e) and the window centred at 1.2 s after memory onset (reading a). All 809 neurons give median accuracies of 0.442 (1.2 s), 0.458 (1.3 s) and 0.452 (median over maintenance windows, reading b), p = 0.76 to 0.87 against 200 scrambled runs; readings (d) and (f) give 0.440 to 0.487, p of 0.54 or more. Leaving out each participant gives quartiles 0.442 to 0.458 (1.2 s), 0.454 to 0.467 (1.3 s) and 0.454 to 0.462 (reading b). With every session's spikes moved by three trials (variant) accuracy is 0.583 to 0.587, p = 0.04 to 0.08, still far from 87.5%. The training sets hold 114 pseudo-trials per split under reading (c) and 190 under reading (d), where the paper gives 145 to 169 (95% of each session's 153 to 178 correct trials); neither reading uses each session's whole correct-trial set. Accuracy as deposited is below the scrambled runs (0.50), which fits the misalignment of spikes and labels suggested by the data check, so this failure may come from the deposited data; moving every session by three trials gives 0.585, far from 87.5%.", "deviations": [ "100 resampling runs and 200 scrambled runs for readings (c) and (e), 50 and 100 for the other readings, and 30 runs per left-out participant without a null, against 500 and 1000 in the paper, for time; the smallest p is 0.005 (0.0099), below the corrected threshold of .0125.", "Window centres are on a 100 ms grid from memory array onset (0.9 to 1.6 s); the paper does not say where the grid starts.", "The SVM is fitted on a precomputed linear kernel (sklearn SVC, C = 1), which is the linear libsvm classifier of the NDT toolbox." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4d-decode-workload-regions", "statement": "Workload was decoded during maintenance from hippocampal neurons (75%) and entorhinal neurons (80%) but not from amygdala neurons.", "source": "Results, 3.4 Decoding of information during the trial, paragraph 1; Figure 4d", "quote": "When pooling neurons from the three anatomical re- gions separately, decoding accuracy was significant for the hippocam- pus (75%, Fig. 4 d column 3) and for the entorhinal cortex neurons (80%, Fig. 4 d column 5), but not for neurons in the amygdala.", "statistics": [ { "about": "hippocampus", "test": "permutation test against scrambled labels", "p": 0.006, "corrected": "Bonferroni over 4 regions", "n": "454 neurons" }, { "about": "entorhinal cortex", "test": "permutation test against scrambled labels", "p": 0.001, "corrected": "Bonferroni over 4 regions", "n": "209 neurons" } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Readings (c), (e), window 1.2 s and regions by tip location (a2) for the measured values. By tip location (a2) hippocampal neurons (467) give 0.467 to 0.481 and entorhinal neurons (196) 0.475 to 0.500 under the window readings, p of 0.43 or more; with the electrode-name fallback (a1) 0.483 to 0.492 (495 neurons) and 0.467 (168), p of 0.60 or more; by electrode name 0.492 to 0.529 (361 neurons) and 0.454 to 0.458 (217 neurons), p of 0.30 or more. The paper reports 75% and 80% with p < .0125. Amygdala neurons are not significant under any region reading (0.458 to 0.475, p of 0.68 or more), as the paper finds, so one of three regions matches. With spikes moved by three trials (electrode name, 100 scrambled runs), hippocampus 0.554 to 0.571 (p of 0.089 or more) and entorhinal cortex 0.542 to 0.571 (p = 0.010 for the median over windows, 0.079 and 0.23 for single windows).", "deviations": [ "100 resampling runs and 200 scrambled runs per region, against 500 and 1000, for time; 30 runs and 100 scrambled runs for the realigned variant." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "decode-performance-not-significant", "statement": "Trial-wise performance (correct against incorrect) could not be decoded from MTL firing in any group (all 55%, hippocampus 55%, entorhinal 55%, amygdala 50%; p from .40 to .56).", "source": "Results, 3.4 Decoding of information during the trial, paragraph 2", "quote": "Decoding accuracy for the prediction of trial-wise performance (cor- rect vs. incorrect trials) was not significant (All MTL: 55%, p = .45, Hipp: 55%, p = .40, Ent: 55%, p = .44, Amg: 50%, p = .56). We could not decode performance with neural firing in the MTL.", "statistics": [ { "about": "all MTL", "test": "permutation test against scrambled labels", "p": 0.45 }, { "about": "hippocampus", "test": "permutation test against scrambled labels", "p": 0.4 }, { "about": "entorhinal cortex", "test": "permutation test against scrambled labels", "p": 0.44 }, { "about": "amygdala", "test": "permutation test against scrambled labels", "p": 0.56 } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_boran_2022.ipynb", "notes": "Window 1.2 s and regions by tip location (a2) for the measured values. With 13 trials per label (the fewest incorrect trials in a session) and 13 cross-validation splits, accuracies are 0.462 to 0.538 for all neurons and each region under every region reading and both window readings, p from 0.27 to 0.77 against 100 scrambled runs; no group reaches p < .0125. The same pipeline does not decode workload (fig4d-decode-workload-all), which the paper decodes at 87.5%, and 13 folds of one trial per label give a test of little demonstrated sensitivity, so this reproduced claim of absence carries little weight.", "deviations": [ "13 cross-validation splits in place of 20, as 13 trials per label cannot be split 20 ways; 50 resampling runs and 100 scrambled runs, for time." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null } ], "findings": [ { "kind": "other", "statement": "The spikes may be misaligned with the trial table. Workload information in the spikes (squared t of low against high workload, summed over units) is at chance in every session when the spikes of trial window i are paired with the label of trial i (p = 0.12 to 0.94 against shuffled labels). Pooled over the 7 sessions with all 192 trials it peaks when they are paired with the label of trial i + 3 (corrected p = 0.005), and decoding workload from all neurons gives 0.46 as deposited, below scrambled labels, against 0.60 with every session moved by three trials. Session by session the shift is supported less: of the 7 complete sessions only P03S1 and P06S1 have p < .05 at a shift of three trials, P02S1 has p = 0.95 there and four others peak at other shifts, and the pooled peak is weighted toward P03S1 and P06S1 (52 and 96 units). In the 9 sessions with fewer trials no single shift carries the information, and the trial table holds no source numbering that would show where trials were left out. A single offset is therefore not established. Within trials, the spikes are consistent with the epochs.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "The location of each unit is the atlas label of the macro contact whose number equals the unit's microwire number (for example the units on wire AHL6 get the label of contact AHL6, in the middle temporal gyrus). The microwires protrude from the tip of the electrode, so these are not the units' locations; 614 of 809 units get a label outside hippocampus, parahippocampal gyrus and amygdala.", "found": null, "status": null }, { "kind": "metadata_insufficient", "statement": "The dandiset does not hold the assignment of units to regions the paper used, nor the location of the microwire tips that Figure 1d shows. By electrode name the units are 361 hippocampal, 217 entorhinal and 226 amygdala; by the atlas label of the tip contact 362 hippocampal, 168 parahippocampal, 146 amygdala and 133 elsewhere; with those 133 placed by the nearest contact with an MTL label (or the electrode name where there is none) 467, 196 and 146; the paper reports 454, 209 and 146. Five units are on bundles (PCL, HL, HR) that the subject descriptions do not list as implanted.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "Participant 12's two sessions are each deposited twice, in four files with the same session start time (2016-12-17 10:38); the pairs are identical, and only the identifiers (session01, session02) tell the sessions apart.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "The response timestamps put 1385 of 3038 responses before the test array appears. The conversion adds the trial start and a 1.7 s event offset to the stored response time; read that way, the stored values are response times from test onset of 0.41 to 3.15 s. The response series does not say what its timestamps mean.", "found": null, "status": null }, { "kind": "selection_undocumented", "statement": "Sessions hold 174 to 192 trials (3038 in all), where the paper and the files' task description state 192 per session. Trial ids run without gaps, and which trials were left out, and why, is not documented.", "found": null, "status": null }, { "kind": "other", "statement": "Spikes are stored only inside trial windows (fixation onset to 2 s after test onset), so nothing between trials, including the inter-trial interval, can be analysed. The fixation period is a fixed 0.9 s in the epochs, where the task description has a jittered black dot.", "found": null, "status": null }, { "kind": "paper_inconsistency", "statement": "The Results give the response-time ANOVA (F = 48.47) on mean response time; on this dataset that F is obtained exactly from per-session median response times (mean gives 61.36), which Figure 1c plots. The 118 ms/item slope is not obtained by any of three readings on the set size (69 to 76); it equals the slope of the median response times on the condition step (1 to 4), 118.1 ms per step.", "found": null, "status": null } ], "reconciliation": null, "verifications": [ { "date": "2026-10-30", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The executed notebook of the run after author-answers with the conclusion cells and the summary removed, the helper module and the claims file, with the paper's full text. The verifier tested the decode and perm_ttest helpers on synthetic data and checked the archive's asset list. Report in transcripts/000575/verifier-1-report.md.", "agreed": 17, "of": 17, "disagreements": [] }, { "date": "2026-10-30", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The full executed notebook of the run after author-revise-1, with the conclusion cells and the summary, the helper module, the claims file, the attempt record and the README, with the paper's full text. The plotted images were not viewed. Report in transcripts/000575/verifier-2-report.md.", "agreed": 17, "of": 17, "disagreements": [] } ] } ], "entries": [ { "name": "000575-boran-2022", "type": "reproduction", "title": "Persistent firing of human MTL neurons in visual working memory (Boran et al. 2022)", "description": "Tests the findings of Boran, Hilfiker, Stieglitz, Sarnthein and Klaver (2022) on dandiset 000575, which holds the trials, answers and sorted units of 16 sessions from 13 epilepsy patients performing a change detection task with 1 to 6 colored squares. It checks the behavior, the neuron counts and firing rates, the maintenance and test neurons, the demixed PCA of the population and the decoding of workload and performance.", "datasets": [ "dandi:000575" ], "papers": [ "10.1016/j.neuroimage.2022.119123" ], "derived_from": [], "notebooks": [ { "path": "reproduce_boran_2022.ipynb", "title": "Reproducing Boran et al. (2022) from dandiset 000575", "executed": "notebooks/000575-boran-2022/reproduce_boran_2022.html" } ] } ] }