{ "dataset": { "archive": "dandi", "id": "000034", "label": "DANDI:000034" }, "page": "datasets/dandi/000034/", "generated": "2026-10-22T21:23:47+00:00", "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.", "outcome_counts": { "reproduced": 0, "partially_reproduced": 1, "not_reproduced": 0, "blocked": 22 }, "outcome_counts_with_outside_data": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_elsewhere": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_links": { "reproduced": 1, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "attempts": [ { "id": "000034-buccino-2020-01", "dataset": { "archive": "dandi", "id": "000034", "version": "0.211030.0713", "doi": "10.48324/dandi.000034/0.211030.0713" }, "key": "dandi:000034", "label": "DANDI:000034", "dataset_version": "0.211030.0713", "paper": "10.7554/elife.61834", "paper_title": "SpikeInterface, a unified framework for spike sorting", "date": "2026-10-22", "kind": "agent", "accountable": "CatalystNeuro", "agent": { "harness": "Claude Code (GitHub Actions)", "model": "claude-opus-5-5", "protocol": "pilot-10" }, "runner": { "platform": "GitHub Actions", "url": "https://github.com/catalystneuro/reproduction-registry/actions/runs/39708723196" }, "tier": "agent_verified", "deprecated": "", "deprecated_on": null, "entry": "000034-buccino-2020", "entry_title": "SpikeInterface spike sorter comparison (Buccino et al. 2020)", "description": "Tests the findings of Buccino, Hurwitz, Garcia, Magland, Siegle, Hurwitz and Hennig (2020) on dandiset 000034, which holds the mouse, simulated, rat and retina recordings the paper sorts and two manual curations of its Kilosort2 output. It checks that the recordings are as described, that the curated and ground-truth spike times share the recordings' clocks, and compares the two curations. The findings about sorter outputs are blocked, as three of the six sorters need MATLAB and a GPU, which the runner lacks.", "notebooks": [ { "path": "reproduce_buccino_2020.ipynb", "title": "Reproducing Buccino et al. (2020) from dandiset 000034", "executed": "notebooks/000034-buccino-2020/reproduce_buccino_2020.html" } ], "outcomes": [ { "claim": "recordings-as-described", "statement": "The dandiset holds the recordings the paper analyses and the files its notebooks name, with the properties the paper and the notebooks give.", "source": "Results, opening paragraph; Data availability; the four notebooks at ee08fb6", "quote": "The datasets are publicly available in NWB format on the DANDI archive", "statistics": [], "kind": "link", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_buccino_2020.ipynb", "notes": "All 15 checks hold. The five files the claims file names are present. The mouse 412804 recording is sub-mouse412804_ecephys.nwb, 27,000,000 frames at 30 kHz (900 s), 248 channels (the notebook's count; the paper says 246 active channels). The simulation lasts 600.00 s at 32 kHz and holds 250 ground-truth units. The rat recording has 384 channels at 30 kHz (270.01 s) and the retina recording 1024 channels at 23199.09 Hz (135.73 s). The curated and ground-truth spike times share the clocks of their recordings on the first seconds read (deepest mean trough at lag 0 in median; within 3 frames for 0.88 and 0.86 of the curated units and 0.63 of the ground-truth units).", "deviations": [ "The paper's notebook loads a local file, mouse412804_probeC_15min.nwb; the dandiset's mouse recording is matched to it by subject and properties, not by name." ], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig1b-unit-counts", "statement": "On the real Neuropixels recording the six sorters return very different numbers of units: Tridesclous the fewest (187) and SpyKING Circus the most (628); HerdingSpikes2 210, Kilosort2 446, IronClust 233 and HDSort 317.", "source": "Results, Spike sorters show low agreement for the same high-density dataset, paragraph 3; Figure 1B", "quote": "with Tridesclous (TDC) finding the least units (187) and SpyKING Circus (SC) finding the most units (628). HerdingSpikes2 finds 210 units; Kilosort2 finds 446 units; IronClust finds 233 units; and HDSort finds 317 units", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the mouse Neuropixels recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig1c-low-agreement", "statement": "Of the 2031 units detected on the real recording, all six sorters agree on only 33, and two or more sorters agree on only 263.", "source": "Results, Spike sorters show low agreement for the same high-density dataset, paragraph 4; Figure 1C", "quote": "Of the 2031 total detected units, all six sorters agree on just 33 of the units ... We also find that two or more sorters agree on just 263 of the total units", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the mouse Neuropixels recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig1d-most-units-unmatched", "statement": "For most sorters more than half of their units match no other sorter; IronClust and Tridesclous are the exceptions.", "source": "Results, Spike sorters show low agreement for the same high-density dataset, paragraph 4; Figure 1D", "quote": "For most sorters, over 50% of the units that they find do not match with any other sorter (with the exceptions of Ironclust and Tridesclous)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the mouse Neuropixels recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig1s2-agreement-scores-high", "statement": "Matched units mostly agree closely: around 80% of the agreement scores of agreed-upon units are 0.8 or higher.", "source": "Results, Spike sorters show low agreement for the same high-density dataset, paragraph 4; Figure 1, figure supplement 2", "quote": "For agreed-upon units, around 80% of the agreement scores are 0.8 or higher, indicating that matched units typically have high spike train agreement", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the mouse recording or the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig1s3-s4-other-recordings-disagree", "statement": "On a rat cortex Neuropixels recording (six sorters) and a mouse retina Biocam recording (the four sorters able to process it, HerdingSpikes2, Kilosort2, IronClust and HDSort) the sorters disagree as much as on the first recording.", "source": "Results, Spike sorters show low agreement for the same high-density dataset, paragraph 5; Figure 1, figure supplements 3 and 4", "quote": "In both cases, we find similar disagreement among the sorters", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the sorters' output on the rat and retina recordings. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2c-sim-unit-counts", "statement": "On the simulated recording with 250 ground-truth units the sorters again return very different numbers of units: Tridesclous 189, HDSort 458, HerdingSpikes2 233, Kilosort2 415, IronClust 283 and SpyKING Circus 343.", "source": "Results, Evaluating spike sorters on a simulated dataset, paragraph 2; Figure 2C", "quote": "The number of detected units range from the 189 units found by Tridesclous to the 458 units found by HDSort. HerdingSpikes2 finds 233 units; Kilosort2 finds 415 units; IronClust finds 283 units; and SpyKING Circus finds 343 units", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2s1a-precision-recall-bias", "statement": "Sorters trade precision against recall differently: HerdingSpikes2, SpyKING Circus and HDSort favour precision, IronClust recall, Kilosort2 scores well on both, and Tridesclous shows no bias.", "source": "Figure 2, figure supplement 1A legend; Results, Evaluating spike sorters on a simulated dataset, paragraph 2", "quote": "Some sorters seem to favor precision (HerdingSpikes, SpyKING Circus, HDSort), others instead have higher recall (Ironclust) or score well on both measures (Kilosort2). Tridesclous does not show a bias towards precision or recall", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2s1b-accuracy-snr", "statement": "Accuracy depends on the SNR of the ground-truth units for every sorter except Kilosort2, which reaches almost perfect accuracy on low-SNR units.", "source": "Results, Evaluating spike sorters on a simulated dataset, paragraph 2; Figure 2, figure supplement 1B", "quote": "Moreover, the accuracy is modulated by the SNR of the ground-truth units for all spike sorters except Kilosort2 which achieves an almost perfect performance on the low-SNR units", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook). The ground-truth SNR alone could be computed from the simulated traces, but the claim relates the sorters' accuracy to it.", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2d-ks2-highest-scores", "statement": "Kilosort2 scores higher than the other sorters for most ground-truth units.", "source": "Results, Evaluating spike sorters on a simulated dataset, paragraph 2; Figure 2D", "quote": "Kilosort2 attains significantly higher scores than the rest of the spike sorters for most ground-truth units", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2e-ks2-unit-types", "statement": "Kilosort2 has the most well-detected units (245) but also many false positive (147) and redundant (21) units, and two overmerged units.", "source": "Results, Evaluating spike sorters on a simulated dataset, paragraph 3; Figure 2E", "quote": "While Kilosort2 has the most well-detected units (245), this comes at the cost of a high percentage of false positive (147) and redundant (21) units ... In both cases two overmerged units are found", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs Kilosort2's output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2e-ks2-curated-unit-types", "statement": "With Kilosort2's own curation, which removes units with an estimated contamination above 20%, false positive and redundant units are fewer but persist: 241 well detected, 93 false positive, 18 redundant and 2 overmerged.", "source": "Results, Evaluating spike sorters on a simulated dataset, paragraph 3 (parenthesis)", "quote": "In that case the number of well-detected units is 241, false positives are 93, and redundant units are 18. In both cases two overmerged units are found", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs Kilosort2's output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig2e-tdc-few-fp-hds-many-fp", "statement": "Tridesclous detects very few false positive or redundant units while finding many well-detected units, and HDSort finds many more false positive units than any other sorter.", "source": "Results, Evaluating spike sorters on a simulated dataset, paragraph 3; Figure 2E", "quote": "Notably, Tridesclous detects very few false positive/redundant units while still finding many well-detected units. HDSort, on the flip side, finds many more false positive units than any other spike sorter", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig3a-sim-low-agreement", "statement": "On the simulated recording only 139 of the 1921 detected units (7.24%) are agreed upon by all six sorters.", "source": "Results, Low-agreement units are mainly false positives, paragraph 1; Figure 3A", "quote": "Again, we observe a large disagreement among the spike sorting outputs with only 139 units of the 1921 total units (7.24%) being in agreement among all sorters", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig3b-sorters-prone-to-low-agreement", "statement": "HDSort, SpyKING Circus and Kilosort2 are more prone to finding low-agreement units than HerdingSpikes2, IronClust and Tridesclous.", "source": "Results, Low-agreement units are mainly false positives, paragraph 1; Figure 3B", "quote": "highlighting that some sorters are more prone to finding low agreement units (HDSort, SpyKING Circus, Kilosort2) than other sorters (HerdingSpikes2, Ironclust, Tridesclous)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig3c-fp-found-by-one-sorter", "statement": "Almost all false positive units are found by a single sorter, while most real units are matched by more than one sorter; false positives found by two or more sorters number 4 for HerdingSpikes2, Kilosort2 and IronClust, 2 for SpyKING Circus and HDSort and 1 for Tridesclous.", "source": "Results, Low-agreement units are mainly false positives, paragraph 2; Figure 3C legend", "quote": "The plots show that (almost) all false positive units are ones that are found by only a single sorter (not matched with any other sorters), while most real units are matched by more than one sorter", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig3s1-ks2-ic-best-pair", "statement": "With two sorters, combining Kilosort2 and IronClust identified true positives most reliably, with false positive and true positive detection rates of 96% and 95%.", "source": "Results, Low-agreement units are mainly false positives, paragraph 2; Figure 3, figure supplement 1 legend", "quote": "For two sorters, the most reliable identification of true positives for this dataset was achieved by combining Kilosort2 and Ironclust (96% and 95% false positive and true positive detection rate, respectively)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig3s1-pairs-remove-fp-four-keep-tp", "statement": "A pair of sorters suffices to isolate almost all false positive units, but with fewer than four sorters a significant fraction of true positive units, found by only one sorter, are wrongly classed as false positives; four or more sorters reliably identify most true positives.", "source": "Results, Low-agreement units are mainly false positives, paragraph 2; Figure 3, figure supplement 1 legend", "quote": "This analysis showed that using a pair of sorters is sufficient to isolate almost all false positive units in each sorter, yet when fewer than four sorter outputs are compared, a significant fraction of true positive units found by only one sorter can be wrongly classified as false positives with this approach", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig3d-snr-rises-with-agreement", "statement": "Ground-truth units with higher SNR are found by more sorters.", "source": "Results, Low-agreement units are mainly false positives, paragraph 3; Figure 3D", "quote": "This shows, as expected, that higher SNR units have higher agreement among sorters", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig3s2-snr-does-not-separate-ks2-fp", "statement": "For Kilosort2, SNR is not a good predictor of false positives: many of its false positive units have an SNR above the mode of the ground-truth SNR.", "source": "Results, Low-agreement units are mainly false positives, paragraph 3; Figure 3, figure supplement 2 legend", "quote": "Many detected false positive units have an SNR above the mode of the ground-truth SNR, indicating that SNR is not a good measure to separate false and true positives in this case", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs Kilosort2's output on the simulated recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4a-curators-agree", "statement": "The two experts' manual curations of the Kilosort2 output agree on most curated units (226 of 351, 64.2%), and 174 Kilosort2 units are discarded by both.", "source": "Results, Consensus units highly overlap with manually curated ones, paragraph 1; Figure 4A", "quote": "most of the curated units (226 out of 351\u201364.2%) are agreed upon by both experts. Notably, 174 units found by Kilosort2 are discarded by both experts", "statistics": [], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_buccino_2020.ipynb", "notes": "Computed from the two curated files under the criterion's second reading (compare_two_sorters of spikecomparison 0.2.6, defaults). Curated 1 has 258 units and curated 2 has 319; 226 are matched and 351 belong to either, 64.39%, so 226 and 351 equal the paper's and 64.39% is within 6.42 of its 64.2%. With a minimum score from 0.3 to 0.8 the matched count goes from 231 to 209, inside its tolerance; the ratio stays inside its tolerance up to 0.7 (59.83%) and falls below it at 0.8 (56.79%, lower bound 57.78%). The criterion fixes the default score of 0.5, so this does not decide the outcome. The count of 174 Kilosort2 units discarded by both curators needs the paper's Kilosort2 run (fig1b-unit-counts, blocked); the curated files hold unit ids and spike times only, with no link to the Kilosort2 clusters (HDF5 tree printed), so it is untested. The criterion gives reproduced when all four values hold under one reading and partially reproduced when 226, 351 and 64.2% hold and 174 does not; with 174 untested the claim is partially reproduced at least, and reproduced only if a Kilosort2 run confirmed the 174. The first reading (compare_multiple_sorters with the Kilosort2 run) could not be run. measured is not given, as the fourth expected value has no measurement.", "deviations": [ "spikecomparison 0.2.6 and spikeextractors 0.8.4 were run on NumPy 2 with the removed aliases np.float, np.int, np.Inf, np.product and np.in1d restored as the builtins and functions they stood for.", "The curated files carry no sampling rate; their spike times were turned into frames at the mouse recording's 30 kHz, as the paper's loader (NwbSortingExtractor) does with the rate it is given." ], "disagreed": true, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4b-single-sorter-match-curated", "statement": "Only a small percentage of each sorter's units match the curated units; IronClust matches best (59.83% and 61.1%), above Kilosort2 (50.67% and 56.25%), from whose output the curations were made.", "source": "Results, Consensus units highly overlap with manually curated ones, paragraph 2; Figure 4B", "quote": "The highest percentage match is actually IronClust which is surprising given that the initial sorting output was curated from Kilosort2\u2019s output (IC \u2229 C1 = 59.83%, IC \u2229 C2 = 61.1%, KS \u2229 C1 = 50.67%, KS \u2229 C2 = 56.25%)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the mouse Neuropixels recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook). The curated sortings are in the dandiset (see fig4a-curators-agree).", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4c-consensus-match-curated", "statement": "Consensus units (found by at least two sorters) match the curated units at above 70% for every sorter; Kilosort2 is highest (84.55% and 89.55%), slightly above IronClust (82.63% and 83.83%).", "source": "Results, Consensus units highly overlap with manually curated ones, paragraph 2; Figure 4C", "quote": "The average match percentage is above 70% for all sorters showing that there is a large agreement between the manually curated outputs and the consensus-based output. Kilosort2 has the highest match (KS c \u2229 C1 = 84.55%, KS c \u2229 C2 = 89.55%), slightly higher than Ironclust (IC c \u2229 C1 = 82.63%, IC c \u2229 C2 = 83.83%)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the mouse Neuropixels recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook). The curated sortings are in the dandiset.", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "fig4d-nonconsensus-rarely-curated", "statement": "Units found by only one sorter rarely match the curated units; the only notable overlap is Kilosort2's (18.58% with C1 and 24.34% with C2).", "source": "Results, Consensus units highly overlap with manually curated ones, paragraph 2; Figure 4D legend", "quote": "The only significant overlap is between Curator one and Kilosort2, with a percent around 18% (KS nc \u2229 C1 = 18.58%, KS nc \u2229 C2 = 24.34%)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "not_runnable", "notebook": null, "notes": "Needs the six sorters' output on the mouse Neuropixels recording. The dandiset holds no sorter output. Kilosort2, IronClust and HDSort are MATLAB programs (Kilosort2 also needs a CUDA GPU); the runner has no MATLAB, GPU or MPI (printed in the notebook) and they cannot be installed from a lock file. Reading time does not add to the block: read chunk by chunk through h5py a whole recording would take hours (notebook table), but one bulk ranged read of the mouse file ran at 72 MB/s in this run, so each recording could be read whole in 1.4 to 6.5 min (notebook). The curated sortings are in the dandiset.", "deviations": [], "disagreed": false, "data": "dataset", "where": null, "proposed": null } ], "findings": [ { "kind": "data_missing", "statement": "The dandiset holds no output of the six spike sorters the paper compares, including the Kilosort2 run that the two experts curated. Every comparison of sorters in the paper, and the comparison of the curations with Kilosort2, can be repeated only by running the sorters again, three of them in MATLAB, and a new Kilosort2 run will not give the units the curators worked on." }, { "kind": "metadata_insufficient", "statement": "The two curated files hold a Units table of ids and spike times only (description 'Autogenerated by NWBFile'), with no sampling rate, no unit labels, no curator name and no link to the Kilosort2 clusters they were curated from; which file is curator 1 and which curator 2 is not stated (the files differ only by session time, 15:55:42 and 15:55:43). Their spike times fall on whole frames at 30 kHz and match the mouse recording's clock." }, { "kind": "metadata_error", "statement": "The retina recording is stored as unsigned 16-bit values centred near 2048 and reaching 4096, consistent with raw counts of a 12-bit converter, while the file states a conversion of 1e-6 V per unit and no offset, which would put every channel near +2 mV. The converter's gain and offset are not in the file." }, { "kind": "metadata_error", "statement": "The mouse 412804 recording stores int16 values with a conversion of 1e-6 V and no gain or channel_conversion per channel. Read that way its band-passed noise SD is about 50 uV and the median curated trough about 455 uV, several times usual Neuropixels levels; at the Neuropixels 1.0 gain of about 0.195 uV per bit the noise would be about 10 uV. The values are consistent with raw counts given a conversion that treats them as microvolts; the file holds nothing to show which." }, { "kind": "metadata_insufficient", "statement": "The electrode tables give location 'unknown' and filtering 'none' for every channel and placeholder group descriptions; the mouse recording has 248 channels where the paper says 246 are active, without marking which; the simulated file does not say which ground-truth cells are excitatory or inhibitory or what noise was added; the archive's species summary lists only house mouse though one recording is from a rat (its file gives 'rattus rattus')." }, { "kind": "other", "statement": "Every raw series is stored in chunks of one channel by 7911 to 26368 frames, so reading a stretch of all channels through h5py fetches one chunk per channel. Streamed from the archive at 0.087 to 0.155 s per chunk in this run, reading a whole recording that way would take 6.2 to 16.9 h; one bulk ranged read of a file ran at 72 MB/s, 1.4 to 6.5 min per recording. These times vary between runs." }, { "kind": "paper_inconsistency", "statement": "The paper gives 226 of 351 curated units as 64.2%; 226/351 is 64.39%." } ], "verifications": [ { "date": "2026-10-22", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The executed notebook of the first run with the conclusion cells and the summary removed, the helper module and the claims file, with the paper's full text; not the attempt record. The verifier checked the asset list and metadata with the archive API and tested to_frames and lag_scan on synthetic input. Report in transcripts/000034/verifier-1-report.md. The cell numbers in the verifier's round-1 reasons refer to that notebook; in the revised notebook the link check is cell 18, the software check cell 21 and fig4a cell 24.", "agreed": 23, "of": 24, "disagreements": [ { "claim": "fig4a-curators-agree", "outcome": "partially_reproduced", "reason": "Cell 21: pairwise compare_two_sorters(C1, C2) with defaults gives 226 (expected 226 \u00b1 22.6), 351 (351 \u00b1 35.1) and 64.39% (64.2 \u00b1 6.42). The 174 needs the Kilosort2 run and is untested. The criterion gives reproduced if 174 holds and partially reproduced if not, so the claim is at least partially reproduced either way. The notebook calls the whole claim blocked, as it depends on fig1b, yet the dependency only touches the 174 (point B1). The verifier could not confirm that the curated Units tables carry no Kilosort2 labels or links, and asked for the HDF5 tree of both curated files and of the mouse recording." } ] }, { "date": "2026-10-22", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The full executed notebook of the revised run, with the conclusions and the summary, the helper module, the claims file, the README and this record, with the paper's full text. Report in transcripts/000034/verifier-2-report.md. Points not tied to one claim, and what was done: cell 0 said every finding but the recordings' presence rests on running six sorters, against its own table, and now names the fig4a curator counts and the three Kilosort2-only claims; cell 13's label asserted microvolts and now says the scale is not checked; the Summary's 'not with microvolts' was softened; the species summary finding is now backed by a print in cell 3; the streaming finding now gives this run's numbers (0.087 to 0.155 s per chunk, 6.2 to 16.9 h, against the verifier's 0.080 to 0.142 s and 5.6 to 15.5 h); and the round-1 record now says its cell numbers refer to the round-1 notebook.", "agreed": 24, "of": 24, "disagreements": [] } ] } ], "entries": [ { "name": "000034-buccino-2020", "type": "reproduction", "title": "SpikeInterface spike sorter comparison (Buccino et al. 2020)", "description": "Tests the findings of Buccino, Hurwitz, Garcia, Magland, Siegle, Hurwitz and Hennig (2020) on dandiset 000034, which holds the mouse, simulated, rat and retina recordings the paper sorts and two manual curations of its Kilosort2 output. It checks that the recordings are as described, that the curated and ground-truth spike times share the recordings' clocks, and compares the two curations. The findings about sorter outputs are blocked, as three of the six sorters need MATLAB and a GPU, which the runner lacks.", "datasets": [ "dandi:000034" ], "papers": [ "10.7554/elife.61834" ], "derived_from": [], "notebooks": [ { "path": "reproduce_buccino_2020.ipynb", "title": "Reproducing Buccino et al. (2020) from dandiset 000034", "executed": "notebooks/000034-buccino-2020/reproduce_buccino_2020.html" } ] } ] }