{ "dataset": { "archive": "dandi", "id": "000469", "label": "DANDI:000469" }, "page": "datasets/dandi/000469/", "generated": "2026-11-05T06:30:02+00:00", "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.", "outcome_counts": { "reproduced": 18, "partially_reproduced": 0, "not_reproduced": 1, "blocked": 1 }, "outcome_counts_with_outside_data": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_elsewhere": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_links": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "attempts": [ { "id": "000469-kyzar-2024-01", "dataset": { "archive": "dandi", "id": "000469", "version": "0.240123.1806", "doi": "10.48324/dandi.000469/0.240123.1806" }, "key": "dandi:000469", "label": "DANDI:000469", "dataset_version": "0.240123.1806", "paper": "10.1038/s41597-024-02943-8", "paper_title": "Dataset of human-single neuron activity during a Sternberg working memory task", "date": "2026-10-30", "kind": "agent", "accountable": "CatalystNeuro", "agent": { "harness": "Claude Code", "model": "claude-opus-5-5", "protocol": "pilot-12" }, "runner": { "platform": "GitHub Actions", "url": "https://github.com/catalystneuro/encore/actions/runs/41057427767" }, "tier": "agent_verified", "deprecated": "", "deprecated_on": null, "entry": "000469-kyzar-2024", "entry_title": "Human single neurons in a Sternberg working memory task (Kyzar et al. 2024)", "description": "Tests the technical validation of Kyzar, Kaminski, Brzezicka, Reed, Chung, Mamelak and Rutishauser (2024) on dandiset 000469, which holds the screening and Sternberg sessions of 21 subjects with 1809 units. It checks the session and unit counts, behaviour, spike sorting quality metrics, and the proportions of concept, maintenance and probe cells, using the permutation tests of the paper's Methods and the paper's parametric code as a variant.", "notebooks": [ { "path": "reproduce_kyzar_2024.ipynb", "title": "Reproducing Kyzar et al. (2024) from dandiset 000469", "executed": "notebooks/000469-kyzar-2024/reproduce_kyzar_2024.html" } ], "outcomes": [ { "claim": "subjects-sessions", "statement": "The dataset holds 41 recording sessions from 21 patients, every patient with a Sternberg session and all but one (P47HMH) with a screening session.", "source": "Methods, Subjects, paragraph 1; Table 1 and its legend", "quote": "In total, 41 recording sessions were completed across 21 patients with intractable localization-related epilepsy (Table 1 ). ... Screening sessions are available for all subjects except for 'P47HMH', for whom screening was not performed.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "The files hold 21 subject IDs, 41 sessions (20 screening, 21 Sternberg) and one subject without a screening session, subject 19 (P47HMH). Subjects 13 and 14 have the native IDs P39CS and P39CS_2, age 26, male, recorded in 2015, with different units and electrodes (subject 13 has 48 and 33 electrodes in its two files, subject 14 has 34 and 34, at different locations); the files do not say whether they are one person recorded twice, so the 21 subjects may be 20 patients. Ages in the files run from 17 to 70, as in Table 1; the text gives 17 to 71.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "sternberg-unit-yield", "statement": "In the Sternberg task 902 units were recorded, 190 in the hippocampus, 259 in the amygdala, 171 in the dACC, 250 in the pre-SMA and 32 in the vmPFC.", "source": "Technical Validation, Data recorded and spike sorting quality metrics, paragraph 1; Figure 3b", "quote": "In the Sternberg task, we recorded a total of 902 neurons (190 hippocampus, 259 Amygdala, 171 dorsal anterior cingulate cortex, 250 pre-supplementary motor area, 32 ventromedial prefrontal cortex) (Fig. 3b ).", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Every count equals the paper's (902 units; 190 hippocampus, 259 amygdala, 171 dACC, 250 pre-SMA, 32 vmPFC). The notebook gives the counts per subject.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "screening-unit-yield", "statement": "In the screening task 907 units were recorded, 449 of them in the MTL.", "source": "Technical Validation, Data recorded and spike sorting quality metrics, paragraph 1; Figure 3a", "quote": "In the screening task, we recorded 907 neurons (Fig. 3a ). Of these, 449 were recorded in the MTL.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "907 screening units, of which 421 in the MTL (159 hippocampus, 262 amygdala) and 486 in the MFC. 421 is within 10% of both readings of the criterion, 449 and 421, and equals the 421 implied by the paper's screening percentages (138 / 0.3278). 449 is the Sternberg MTL count (190 + 259), so the text gives the Sternberg number for the screening task.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "behavior-accuracy", "statement": "Subjects answered on average 88.73% of Sternberg trials correctly.", "source": "Technical Validation, Behavior, paragraph 1; Figure 1d", "quote": "During the working memory task (Fig. 1b ), subjects answered on average 88.73% of trials correctly (Fig. 1d ).", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Mean over the 21 sessions 88.73% (s.e.m. 1.25, sessions 75.00 to 95.56%).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "behavior-rt-load", "statement": "Reaction times grew with memory load; a repeated-measures ANOVA of per-session median reaction times found a load effect.", "source": "Technical Validation, Behavior, paragraph 1; Figure 1c", "quote": "Accuracy and reaction time varied as a function of load as expected (Fig. 1c ), with higher loads resulting in slower and less accurate answers (F 2,40 = 7.92, P = 0.0013, repeated-measures ANOVA of median reaction times).", "statistics": [ { "test": "repeated-measures ANOVA of median reaction times", "statistic": "F(2,40) = 7.92", "p": 0.0013, "n": "21 Sternberg sessions (df 2 and 40)" } ], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "On correct trials the mean of the session medians is 1.055, 1.130 and 1.198 s for loads 1 to 3, and the load effect of the additive session by load ANOVA is F(2,40) = 7.92, p = 0.0013, the paper's values. On all trials (variant) F(2,40) = 4.08, p = 0.024. 17 of 21 sessions are slower at load 3 than at load 1.", "deviations": [ "The repeated-measures ANOVA is computed as the code's anovan model (session random, load fixed, no interaction), whose F test of load is the additive two-way ANOVA's; the correct-trial filter is the code's, as the criterion says." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "behavior-accuracy-load", "statement": "Accuracy was lower at higher memory loads.", "source": "Technical Validation, Behavior, paragraph 1; Figure 1c", "quote": "Accuracy and reaction time varied as a function of load as expected (Fig. 1c ), with higher loads resulting in slower and less accurate answers", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Mean accuracy over sessions 91.22% (load 1), 88.62% (load 2) and 86.35% (load 3), falling monotonically. The notebook gives the per-subject values; the paper reports no test.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "isi-violations", "statement": "Across Sternberg units, refractory period violations (ISIs shorter than 3 ms) were 0.41% of ISIs on average, with a standard deviation of 0.50%.", "source": "Technical Validation, Data recorded and spike sorting quality metrics, paragraph 2; Figure 3d", "quote": "On average across all neurons in the Sternberg task, ( i ) refractory period violations were 0.41% \u00b1 0.50% of interspike intervals (ISIs) (Fig. 3d )", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Over all 902 units (main reading) the mean is 0.428% (within tolerance) and the s.d. 0.625% (outside 0.50 plus or minus 0.05). With the code's filter (units below 4%), which leaves out one unit at 11.74%, the values are 0.416% and 0.498%, both within tolerance. The claim holds on the code's reading only; the measured values are that reading's.", "deviations": [ "The match needs the code's exclusion of units with 4% or more violations, which the paper does not state; it removes one unit." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "firing-rate", "statement": "Sternberg units fired at 2.55 Hz on average, ranging from 0.11 to 33.82 Hz.", "source": "Technical Validation, Data recorded and spike sorting quality metrics, paragraph 2; Figure 3e", "quote": "(ii) average firing rate was 2.55 Hz, with a range of 0.11\u201333.82 Hz (Fig. 3e )", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Mean 2.546 Hz, minimum 0.108 Hz, maximum 33.824 Hz over 902 units (median 1.387 Hz).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "projection-distance", "statement": "The pairwise projection distance between units isolated on the same wire was 13.77 plus or minus 7.32 s.d. of the signal.", "source": "Technical Validation, Data recorded and spike sorting quality metrics, paragraph 2; Figure 3i", "quote": "( iii ) the pairwise projection distance in clustering space between neurons isolated on the same wire was 13.77 \u00b1 7.32 (projection test; in units of s.d. of the signal)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "734 of 902 units have a non-zero stored projection distance; their mean is 13.774 and s.d. 7.325 (range 3.72 to 54.13).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "waveform-snr", "statement": "The ratio between the s.d. of the noise and the peak amplitude of the mean waveform (an SNR) was 6.62 plus or minus 3.63 across Sternberg units.", "source": "Technical Validation, Data recorded and spike sorting quality metrics, paragraph 2; Figure 3g,h", "quote": "( iv ) the ratio between the s.d. of noise and the peak mean waveform amplitude was 6.62 \u00b1 3.63 (Fig. 3g,h )", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Reading (a), the stored peak SNR, gives 6.625 plus or minus 3.632, within tolerance. Reading (b), the stored mean SNR, gives 1.647 plus or minus 0.954 and does not match.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "cv2", "statement": "The modified coefficient of variation of the ISIs (CV2) was 0.94 plus or minus 0.14 across Sternberg units.", "source": "Technical Validation, Data recorded and spike sorting quality metrics, paragraph 2; Figure 3f", "quote": "(v) the modified coefficient of variation of the ISIs (CV2 52 ) was 0.94 \u00b1 0.14 (Fig. 3f )", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "CV2 0.938 plus or minus 0.138 over 902 units, none undefined.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "isolation-distance", "statement": "The median isolation distance of Sternberg units was 31.14.", "source": "Technical Validation, Data recorded and spike sorting quality metrics, paragraph 2; Figure 3j", "quote": "(vi) median isolation distance 53 was 31.14 (Fig. 3j )", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Median 31.138 over the 676 units with a defined isolation distance (226 are NaN).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "screening-concept-mtl", "statement": "In the screening task, 138 MTL units (32.78%) qualified as concept cells.", "source": "Technical Validation, Proportion of selective cells, paragraph 2", "quote": "Screening task: Of the MTL cells, 138 (32.78%) qualified as concept cells (Fig. 4 shows examples).", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Of 421 MTL screening units, 140 (33.25%) pass the permuted ANOVA and the permutation t-test, with the second test two-sided and one-sided alike (the same cells); both readings are within tolerance. Over the text's 449 the percent would be 31.18%, also within tolerance. Re-testing units near p = 0.05 with 20000 permutations gives 139 (33.02%). The code's parametric tests give 138 (32.78%), the paper's numbers exactly. Concept cells come from 16 of the 19 subjects with MTL units; subjects 13 and 14 hold 56 of the 140.", "deviations": [ "2000 permutations, the number in the code's commented permutation alternative; the paper gives none. The p-value is (exceedances + 1) / (permutations + 1), as the code's permutationTest.m.", "The permutation t-test compares the difference of means with label permutations, as the code's permutationTest.m does; the Methods name a \"permutation t-test\" without saying which statistic is permuted." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "screening-concept-mfc", "statement": "In the screening task only 26 MFC units (5.35%) were concept cells, far fewer than in the MTL, so that concept cells are largely found in the MTL.", "source": "Technical Validation, Proportion of selective cells, paragraph 2", "quote": "In contrast, only 26 (5.35%) of cells in the MFC were concept cells, reproducing our earlier finding that concept cells are largely only found in the MTL.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Of 486 MFC screening units, 27 (5.56%) are concept cells under both readings, within tolerance of 26 and 5.35%, and far below the MTL's 33.25%. The count is near the upper bound of the tolerance (28.6); with three other seeds it is 29, 28 and 27, and the 29 (5.97%) would be outside. Re-testing the 142 MFC units with a p-value between 0.02 and 0.10 with 20000 permutations gives 26 (5.35%), within tolerance. The code's parametric tests give 26 (5.35%), the paper's numbers.", "deviations": [ "As for screening-concept-mtl (2000 permutations; difference of means permuted). The outcome at 2000 permutations depends on the seed for this claim, because the count lies near the tolerance bound; the 20000-permutation variant settles it within tolerance." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4-screening-examples", "statement": "Five example screening units (Fig. 4a to e) are concept cells.", "source": "Figure 4 legend", "quote": "Shown are five ( a \u2013 e ) neurons classified as concept cells.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "All five units pass the concept cell test under both readings (permuted ANOVA p 0.0005 to 0.028, second test p at most 0.001) and under the code's tests. Four are in the amygdala and one in the hippocampus.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "sternberg-concept-mtl", "statement": "In the Sternberg task, 94 MTL units (20.94%) qualified as concept cells during the encoding period.", "source": "Technical Validation, Proportion of selective cells, paragraph 3", "quote": "Working memory task: In the MTL, 94 (20.94%) neurons qualified as concept cells during the encoding period (Fig. 5 shows examples).", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Of 449 MTL Sternberg units, 96 (21.38%) are concept cells with a two-sided second test and 100 (22.27%) with a one-sided one; both are within tolerance of 94 and 20.94%. The code's parametric tests give 94 (20.94%), the paper's numbers exactly. With three other seeds the counts are 95 to 97 (two-sided) and 99 to 100 (one-sided), all within tolerance. Over all areas the counts are 123, 128 and 119.", "deviations": [ "As for screening-concept-mtl (2000 permutations; difference of means permuted)." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "persistent-activity", "statement": "As a group, concept cells fired more during the maintenance period when their preferred image was held in memory than when it was not (4.76 against 2.67 Hz).", "source": "Technical Validation, Proportion of selective cells, paragraph 3", "quote": "As a group, concept cells continued to fire during the maintenance period if their preferred stimulus was held in WM (4.76 \u00b1 4.71 Hz vs. 2.67 \u00b1 3.66 Hz; p = 1.38 \u00d7 10 \u221221 for pref. vs. non-pref. trials, paired one-sided t-test).", "statistics": [ { "test": "paired one-sided t-test, preferred against non-preferred trials", "p": 1.38e-21 } ], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "In the maintenance window (0 to 2500 ms), under both cell readings, both trial readings and every concept cell set, concept cells fire more on preferred trials (every paired one-sided p below 1e-6), but the preferred means are 2.57 to 3.13 Hz against the paper's 4.76, and no reading puts the four values within tolerance. The measured values are reading (a)(i) with the two-sided concept test, MTL cells, preferred image in any encoding period (96 cells, 2.567 plus or minus 3.768 against 2.116 plus or minus 3.671 Hz, p = 3.9e-08). The variant in the Encoding 1 window (200 to 1000 ms) with preferred as the Encoding 1 image reproduces the paper's numbers exactly for the code's 119 parametric concept cells in all areas (4.764 plus or minus 4.709 against 2.673 plus or minus 3.656 Hz), and within tolerance for the permutation concept cells in all areas (123 cells, 4.631 against 2.588 Hz, p = 8.6e-22; 128 cells, 4.748 against 2.710 Hz, p = 1.3e-22). The paper's values therefore appear to be encoding responses. The direction the statement gives (more firing in maintenance when the preferred image is held) holds; the reported magnitudes do not. Leaving out the three trials with implausible timestamps (findings) and recomputing the concept cells (97 and 102 MTL cells, 94 parametric) gives maintenance-window preferred means of 2.57 to 3.10 Hz, no maintenance reading within tolerance, and 4.768 plus or minus 4.712 against 2.674 plus or minus 3.656 Hz for the parametric Encoding 1 variant, so these trials do not change the outcome.", "deviations": [ "The concept cells are those of sternberg-concept-mtl under its two readings; the code's parametric set is shown too." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig5-sternberg-examples", "statement": "Two example amygdala units (Fig. 5a,b) are concept cells in the Sternberg task that also show persistent activity during maintenance.", "source": "Figure 5 legend", "quote": "( a,b ) Example responses from 2 neurons classified as concept cells in the Sternberg task that also exhibited persistent activity during the maintenance period. Both cells were recorded in the amygdala.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Both units (subject 7 unit 9, subject 16 unit 16, 1-based) are amygdala concept cells under both readings and the code's tests (all p = 0.0005), with the same preferred image (ID 5). Their mean maintenance rate is higher on trials that showed the preferred image (5.05 against 1.43 Hz, and 5.73 against 4.17 Hz).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "maintenance-cells-mfc", "statement": "In the MFC, 144 units (31.79%) qualified as maintenance cells.", "source": "Technical Validation, Proportion of selective cells, paragraph 3", "quote": "In the MFC, 144 (31.79%) cells qualified as maintenance cells and 122 (26.93%) as probe cells.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_kyzar_2024.ipynb", "notes": "Of 453 MFC Sternberg units, label shuffling gives 140 (30.91%, non-preferred as Encoding 1) and 139 (30.68%, preferred not shown), sign flipping 144 (31.79%) and 143 (31.57%); every concept cell set gives the same counts. All are within tolerance of 144 and 31.79%. Counting permuted values equal to the observed one as exceedances (on the same permutations) leaves label shuffling at 140 and 139 and gives 142 and 141 for sign flipping, all within tolerance. The code's paired t-tests give 144 (31.79%) and 143.", "deviations": [ "2000 permutations, from the code's comment; the p-value as for screening-concept-mtl. Exceedances are counted with a strict inequality, as permutationTest.m does, which with discrete spike counts makes the tests slightly anti-conservative; counting ties is shown as a variant." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "probe-cells-mfc", "statement": "In the MFC, 122 units (26.93%) qualified as probe cells.", "source": "Technical Validation, Proportion of selective cells, paragraph 3", "quote": "In the MFC, 144 (31.79%) cells qualified as maintenance cells and 122 (26.93%) as probe cells.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "methods_underspecified", "notebook": "reproduce_kyzar_2024.ipynb", "notes": "All four readings were run on the 453 MFC units. Sign flipping gives 118 (26.05%, pooled encoding rate) and 111 (24.50%, Encoding 1), within tolerance; label shuffling gives 109 (24.06%) and 104 (22.96%), outside it. With three other seeds label shuffling with Encoding 1 stays outside (104 or 105), label shuffling with the pooled rate is just inside (110 or 111, bound 109.8) and sign flipping stays inside (110 to 124), so the readings disagree whatever the seed, and the criterion blocks the claim. Counting ties as exceedances on the main seed gives 108 and 103 for label shuffling and 118 and 111 for sign flipping, the same disagreement. The code's paired one-sided t-tests give 122 (26.93%), the paper's numbers. What is open is whether the Methods' \"permutation t-tests\" between probe and encoding or maintenance rates permute rates between the two periods as independent samples or keep each trial's pair (the code's own tests are paired t-tests, and its commented permutationTest.m is unpaired), and which encoding periods form the encoding rate. The passages are Methods, \"Cell selection, probe cells\" and the code's NWB_calcSelective_SB.m. The question is put to the paper's authors (findings).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null } ], "findings": [ { "kind": "paper_inconsistency", "statement": "The text gives 449 MTL units for the screening task. The screening files hold 421 MTL units (159 hippocampus, 262 amygdala), which agrees with the paper's own screening percentages (138 is 32.78% of 421); 449 is the Sternberg MTL count.", "found": null, "status": null }, { "kind": "paper_inconsistency", "statement": "The persistent activity values in the text (4.76 plus or minus 4.71 against 2.67 plus or minus 3.66 Hz, preferred against non-preferred trials \"during the maintenance period\") are reproduced exactly by the rates 200 to 1000 ms after Encoding 1 onset of all 119 concept cells of the code's parametric test, split by whether the Encoding 1 image is the preferred one. In the maintenance window the same cells fire at 3.05 against 2.70 Hz. The released code computes these Encoding 1 rates under the names hzPref and hzNonPref, with a mask that, as written, selects every trial as preferred (per the claims file's reading of NWB_calcSelective_SB.m).", "found": null, "status": null }, { "kind": "methods_unavailable", "statement": "Question for the authors. The Methods name permutation tests for the concept, maintenance and probe cells, while the released code runs parametric tests (anovan, paired t-tests) and keeps the permutation versions in comments; the parametric tests reproduce every published count exactly here (138, 26, 94, 144, 122). Were the published counts computed with the parametric tests? If permutation tests were used, how many permutations, and were the probe and maintenance comparisons paired by trial (sign flipping) or unpaired (label shuffling)? For the probe cells the choice decides whether the count matches (118 or 111 paired, 109 or 104 unpaired, against 122), and the encoding rate (all encoding periods shown, or Encoding 1 only) is also not stated.", "found": null, "status": null }, { "kind": "selection_undocumented", "statement": "The reported ISI violation spread (0.41 plus or minus 0.50%) is matched only after leaving out units with 4% or more violations, as NWB_QA_graphs.m does (one unit, at 11.74%); over all units it is 0.43 plus or minus 0.63%. The reaction time ANOVA uses correct trials only, as the code does; neither filter is stated in the paper.", "found": null, "status": null }, { "kind": "format_error", "statement": "Four units have spike times out of order, each with one backward step. Screening files of subject 11 (units 26 and 32, steps of 9.8 and 12.4 ms) and subject 5 (unit 18, 0.9 ms), and the Sternberg file of subject 2 (unit 22, 0.3 ms). NWB expects spike times in ascending order.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "Column descriptions in the units and trials tables are wrong. `waveforms_mean_proj_dist` is described as \"Cluster Isolation distance, computed using all waveforms in the cluster\" (the description of `waveforms_isolation_distance`), and `loadsEnc3_PicIDs` as \"Picture ID for Enc1 loads\". Every column of the electrodes table but `id` is described as \"my description\".", "found": null, "status": null }, { "kind": "metadata_insufficient", "statement": "Subjects 13 and 14 have the native IDs P39CS and P39CS_2, the same age (26), sex and recording year, and Table 1 gives the same diagnosis. Neither the files nor the paper say whether they are the same patient, so the count of 21 patients may be 20.", "found": null, "status": null }, { "kind": "other", "statement": "Two Sternberg trials have implausible timestamps. In one trial of subject 9 (load 1) Encoding 1 lasts 2.5 s and the maintenance and probe onsets fall at the same time (529.288 s), so it has no maintenance period. In another trial of subject 9 (load 3) the fixation cross and Encoding 1 onsets are 0.03 ms apart (1189.469 s), so the baseline window falls on the image. In one trial of subject 21 the fixation cross lasts 14.07 s, against the 0.9 to 1 s the paper describes (the 1st and 99th percentiles over trials are 0.93 and 1.20 s). All three are included in the analysis, as the code includes them; leaving them out does not change the persistent activity result.", "found": null, "status": null } ], "reconciliation": null, "verifications": [ { "date": "2026-10-30", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "Round 1. The first executed notebook with the conclusion cells and the summary removed, the helper module and the claims file, with the paper. The verifier queried the archive for the pinned version and tested helper functions on synthetic inputs. It gave its own outcome for each claim; all 20 agree with the record. It found eight problems, none affecting a conclusion.", "agreed": 20, "of": 20, "disagreements": [] }, { "date": "2026-10-30", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "Round 2. The full executed notebook with conclusions and summary, the helper module, the claims file, the attempt record and the README, with the paper. The verifier could not run code (numpy not installed) and read the new _exceed helper. It agrees with all 20 outcomes and found no misreading of the claims file. It listed statements the printed outputs do not support; those not tied to one claim are answered here. The metadata_error finding and the summary state column descriptions no cell printed (a new cell after the clock check now prints them, which also shows the electrodes id column has an empty description; the finding says so). The other finding said 99% of trials where cell 5 prints the 1st and 99th percentiles (corrected). The README run time was not shown (the last cell now prints it, 3.6 min).", "agreed": 20, "of": 20, "disagreements": [] } ] } ], "entries": [ { "name": "000469-kyzar-2024", "type": "reproduction", "title": "Human single neurons in a Sternberg working memory task (Kyzar et al. 2024)", "description": "Tests the technical validation of Kyzar, Kaminski, Brzezicka, Reed, Chung, Mamelak and Rutishauser (2024) on dandiset 000469, which holds the screening and Sternberg sessions of 21 subjects with 1809 units. It checks the session and unit counts, behaviour, spike sorting quality metrics, and the proportions of concept, maintenance and probe cells, using the permutation tests of the paper's Methods and the paper's parametric code as a variant.", "datasets": [ "dandi:000469" ], "papers": [ "10.1038/s41597-024-02943-8" ], "derived_from": [], "notebooks": [ { "path": "reproduce_kyzar_2024.ipynb", "title": "Reproducing Kyzar et al. (2024) from dandiset 000469", "executed": "notebooks/000469-kyzar-2024/reproduce_kyzar_2024.html" } ] } ] }