{ "dataset": { "archive": "dandi", "id": "000231", "label": "DANDI:000231" }, "page": "datasets/dandi/000231/", "generated": "2026-11-05T07:41:23+00:00", "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.", "outcome_counts": { "reproduced": 23, "partially_reproduced": 0, "not_reproduced": 6, "blocked": 10 }, "outcome_counts_with_outside_data": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_elsewhere": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_links": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "attempts": [ { "id": "000231-rodgers-2022-01", "dataset": { "archive": "dandi", "id": "000231", "version": "0.220904.1554", "doi": "10.48324/dandi.000231/0.220904.1554" }, "key": "dandi:000231", "label": "DANDI:000231", "dataset_version": "0.220904.1554", "paper": "10.1038/s41597-022-01728-1", "paper_title": null, "date": "2026-10-29", "kind": "agent", "accountable": "CatalystNeuro", "agent": { "harness": "Claude Code", "model": "claude-opus-5-5", "protocol": "pilot-12" }, "runner": { "platform": "GitHub Actions", "url": "https://github.com/catalystneuro/encore/actions/runs/40812211322" }, "tier": "agent_verified", "deprecated": "", "deprecated_on": null, "entry": "000231-rodgers-2022", "entry_title": "Behavioral, videographic and neural dataset on object recognition in mice (Rodgers 2022)", "description": "Tests the statements of the data descriptor Rodgers (2022, Sci Data 9, 620) on dandiset 000231, the 115 sessions of 15 mice doing whisker-based shape discrimination or detection, with whisker tracking from 200 frames/s video and, in 44 sessions, barrel cortex recordings. It checks the counts, the file layout, the NWB contents, the trial logic, the shared timebase, the unit and electrode tables and the inclusion criteria against the files.", "notebooks": [ { "path": "reproduce_rodgers_2022.ipynb", "title": "Reproducing Rodgers (2022) from dandiset 000231", "executed": "notebooks/000231-rodgers-2022/reproduce_rodgers_2022.html" } ], "outcomes": [ { "claim": "dataset-size", "statement": "The dataset holds 15 mice, 115 sessions, 18,514 trials and 63,979,800 video frames, which is 88.9 hours of video.", "source": "Methods, High-speed videography and whisker tracking, paragraph 2; Background & Summary, paragraph 5", "quote": "The size of the dataset ... 15 mice, 88.9 hours, 115 sessions, 18,514 trials, 63,979,800 frames", "statistics": [], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "Mice (15 subject folders) and sessions (115 NWB files) match. The video frames match exactly under reading (a), 63,979,800 frames or 88.86 h; reading (b), the longest raw pose series of each session, gives 63,082,162 (-1.40%). The trials match under neither reading, 32,350 rows (+74.7%) under (a) and 22,905 with ignore_trial False (+23.7%) under (b), both outside 10%, so the criterion gives not reproduced. Other counts were tried for information (random trials 24,776; random and not spoiled 24,590; ignore_trial False with Table 6 stimuli 21,752; ignore_trial False in sessions without optogenetics 17,287); none gives 18,514.", "deviations": [ "Video frame counts are read from each file's duration tag at the nominal 30 frames/s, not by decoding 2 TB of video. Every duration is a whole number of seconds; the container duration equals the tag in all 115 files. No video stream reports its own frame count (0 of 115), so the tag is the only count the headers give.", "Reading (b) takes the longest of the four whiskers' raw pose series in each session, as the criterion's N_frames differs between whiskers." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "neural-sessions", "statement": "Neural recordings from barrel cortex were made in 44 of the 115 sessions.", "source": "Methods, Neural recordings during behavior, paragraph 1", "quote": "For a subset (44/115) of the behavioral sessions that we report here, we also recorded neuronal activity in identified columns of the barrel cortex", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "44 files are named with ecephys, and the same 44 hold the extracellular array recording and a non-empty units table.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "neural-hours", "statement": "The dataset includes 33.8 hours of 64-channel neural recording at 30 kHz.", "source": "Background & Summary, paragraph 5", "quote": "includes 88.9 hours of behavioral video at 200 frames per second and 33.8 hours of 64-channel neural recording at 30 kHz", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "Reading (b), the main Open Ephys recordings under sourcedata, gives 33.770 h over 44 sessions, within 0.05 h of 33.8; each session has 64 channel files of equal length. The record headers of CH1 show a second, shorter segment in 7 sessions (0.48 to 0.51 s at the start of 6 files, and in KF134 2018-02-10 a separate recording, number 0, of 39.73 s before the main one, number 1); these 42.8 s are left out. Nothing printed shows they are the test recordings the paper mentions, which it says were a few seconds long, while six of the seven are 14 or 15 records under the main segment's recording number; leaving them out is a choice of this attempt. The whole files give 33.782 h, also within, so the outcome does not depend on it. Reading (a), the span of the NWB recordings, gives 33.656 h (-0.42%). The NWB files hold 56 to 64 channels.", "deviations": [ "The samples of each Open Ephys file are computed from its size (1024-byte header, 2070-byte records of 1024 samples). To leave out segments apart from the main recording, such as the test recordings the Data Records mention (\"In some cases the Open Ephys file includes test recordings that were just a few seconds long\"), segments are found from the first and last record headers and a bisection on the record timestamps of channel CH1, which splits each file where the timestamps break; the longest contiguous segment is taken as the main recording." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "task-rules", "statement": "In discrimination sessions convex shapes are rewarded on the right and concave on the left; in detection sessions either shape is rewarded on the right and catch trials with no shape on the left.", "source": "Methods, Two behavioral tasks, paragraph 1", "quote": "Shape discrimination mice were trained to lick right for convex shapes and left for concave shapes. Shape detection mice were trained to lick right for either shape (concave or convex) and to lick left on catch trials when no shape was presented at all", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "The task assigned by the rule matches the experiment description in all 115 sessions (30 detection, 85 discrimination). Over trials to analyse with Table 6 stimuli, 21,752 of 21,752 follow the rule, in every session; all 32,350 trials follow it, including 1,533 with flatter shapes given the rule of their shape. The metadata task is taken from the experiment description, and from the keywords only where the description names none, which never happens; the keywords name \"shape discrimination\" in every file, detection sessions included. The task given by \"nothing\" alone also agrees with the description in all 115 sessions.", "deviations": [ "The files hold stimuli \"flatter convex\" and \"flatter concave\", which Table 6 does not list. The rule is judged on the Table 6 stimuli, and the flatter shapes are shown apart with the rule of their shape." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "nothing-detection-only", "statement": "The stimulus \"nothing\" occurs only in the detection task.", "source": "Data Records, NWB file, trials table, Table 6 (stimulus)", "quote": "'nothing' was only used in the detection task, not in the discrimination task", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "The experiment description names the task in every file. All 3,749 \"nothing\" trials are in the 30 detection sessions; none of the 85 discrimination sessions has one.", "deviations": [ "The criterion allows the task from the experiment description or the keywords. The description is used. The keywords include \"shape discrimination\" in all 115 files, the 30 detection sessions included, so they cannot name the task." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "one-task-per-mouse", "statement": "No mouse was trained on both tasks.", "source": "Methods, Two behavioral tasks, paragraph 1", "quote": "No mice were trained on both tasks", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "Each of the 15 mice has sessions of one task only, 10 discrimination and 5 detection, by the rule and by the experiment description.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "servo-positions", "statement": "The shape stops at one of three positions, close, medium or far.", "source": "Methods, Two behavioral tasks, paragraph 2; Table 6 (servo_position)", "quote": "The shape stopped at one of three different positions (close, medium, or far)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "Trials with a shape take exactly close (9,564), medium (9,494) and far (9,543), and all 115 sessions hold the three. Nothing trials also carry a servo position.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "response-window-timing", "statement": "The response window opens 2 s after the servo motor starts moving the shape, which reaches its final position 0.8 to 0.4 s before the window opens, depending on the position.", "source": "Methods, Two behavioral tasks, paragraph 3", "quote": "which began exactly 2 seconds after the linear servo motor started moving the shape toward the mouse ... reached its final position between \u22120.8 s and \u22120.4 s, depending on whether the final position was close, medium, or far", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": "reproduce_rodgers_2022.ipynb", "notes": "No motor start or stop time is in the files; the trials table's only servo column is servo_position. For information, the window opens a median 3.023 s after start_time (5th to 95th percentile 3.020 to 3.443 s), the same for each position.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "choice-time-first-lick", "statement": "The choice time is the first lick after the response window opens, and on spoiled trials with no lick it is set 45 s after the window opens.", "source": "Data Records, NWB file, trials table, Table 6 (choice_time)", "quote": "choice_time (float) The time of the choice lick, which is the first lick after the response window opened. On spoiled trials with no lick, this value is 45 seconds after the response window opened", "statistics": [], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "On 32,085 non-spoiled trials, the first lick at or after the window opening is within 5 ms of choice_time on 77.6% (78.0% allowing a lick 5 ms early), below 99% in every session (lowest 52.5%); its side matches choice on 99.1%. On the 7,185 trials that do not match, choice_time is 0 to 5 ms after the window opening on 7,183 of them, a lick in the 100 ms before the opening precedes it on 84.8%, and a lick lies within 5 ms of choice_time on only 5.7%. This suggests that a lick already under way when the window opens is taken as the choice at the opening, which the definition does not say. Over all 32,085 non-spoiled trials choice_time is 0 to 5 ms after the opening on 7,848 (24.5%), 7,183 of them mismatched. On all 265 spoiled trials choice_time is NaN, not 45 s after the window.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "outcome-consistency", "statement": "The first lick in the response window decides the outcome; a choice on the rewarded side is correct, any other choice is an error, and a trial with no choice is spoiled.", "source": "Methods, Two behavioral tasks, paragraph 4; Table 6 (outcome, choice)", "quote": "Only the first lick (left or right) during the response window determined the trial outcome (correct or incorrect)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "On all 32,350 trials outcome is correct exactly when choice equals rewarded_side, error when it is the other side, and spoil exactly when choice is nogo; the 46 direct-delivery trials are consistent too.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "reward-and-timeout", "statement": "Correct licks were rewarded with water, and incorrect licks were punished with a timeout, typically 9 s long.", "source": "Methods, Two behavioral tasks, paragraph 4", "quote": "Correct licks were rewarded with water and incorrect licks were punished with a timeout, typically 9 seconds long", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "methods_underspecified", "notebook": "reproduce_rodgers_2022.ipynb", "notes": "The criterion's two readings of the timeout disagree, and it blocks the claim in that case. Over trials to analyse, the median of stop_time minus choice_time is 12.21 s on error trials (reading a, outside 9 +- 0.9 s) and 3.88 s on correct trials, so the difference is 8.34 s (reading b, within); per session the difference is 7.04 to 8.54 s. The reward part also fails as computed, with a reward on 8.5% of the 5,130 error trials over both sides. It holds on the left (reward on 100% of 8,917 correct trials and on none of 2,497 error trials) but not on the right, where the reward series is identical to the right lick series in every file and a \"reward\" falls on 16.6% of error trials; there it counts licks, so right rewards cannot be tested, a defect of the deposit (findings). Which interval the paper calls the timeout is a question for the paper's authors (findings).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "forced-alternation-start", "statement": "Every session began with 45 trials of Forced Alternation, which are non-random trials, before random trials.", "source": "Methods, Session control and trial types to be dropped from analysis, paragraph 2", "quote": "Even in fully trained mice, we found it best to begin every session with 45 trials of Forced Alternation, before switching to random trials", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "In the 110 sessions whose table holds trials numbered 0 to 44, all 4,229 such trials are non-random and the first random trial is number 45. The other 5 tables begin after trial 44; their first random trials are 61, 76, 88, 102 and 124.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "ignore-trial-definition", "statement": "A trial is marked ignore_trial when it is direct delivery, non-random, spoiled, optogenetic or otherwise munged.", "source": "Data Records, NWB file, trials table, Table 6 (ignore_trial)", "quote": "ignore_trial (bool) This is True if 'direct_delivery' is True, or 'stim_is_random' is False, or outcome is 'spoil', or 'optogenetic' is True, or the trial was otherwise munged", "statistics": [], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "All 46 direct-delivery, 7,574 non-random and 2,482 optogenetic trials have ignore_trial True, but 160 of the 265 spoiled trials (random, not optogenetic, in 50 sessions) have it False. ignore_trial equals \"stim_is_random False or optogenetic\" on every trial, and no trial is ignored without one of the four flags (no munged trials are marked).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "trials-table-structure", "statement": "Each trial's stop time equals the next trial's start time, and the table's index is the trial number.", "source": "Data Records, NWB file, trials table, Table 6 and the paragraph after it", "quote": "stop_time (float) Stop time of the trial. (Equal to the start time of the next trial.) ... The index of this DataFrame is called \"id\" and is identical to the field \"trial\"", "statistics": [], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "id equals trial on all 32,350 rows. stop_time equals the next start_time within 1 ms on 32,230 of 32,235 consecutive pairs; the 5 others, in 4 sessions, are all where the trial numbers skip, with gaps of 7.0 to 68.0 s. The criterion asks for every row, so it is not reproduced. Read as the next trial run, the statement holds on all 32,230 pairs whose trial numbers are consecutive (variant). The paper mentions only trials dropped before the video began, at the start of a table. Whether the criterion should compare the next trial run is asked of a reader (questions/000231.yaml); the outcome is tested on the claims file as it is.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "video-timebase", "statement": "Times in the NWB file are in a common video timebase in which frame F falls at exactly F/200 s.", "source": "Methods, Alignment of multiple data streams, paragraph 4", "quote": "Since the camera ran at 200 frames per second, frame F in the video is defined as occurring at exactly time F/200 in the common video timebase", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "All processed and pose timestamps of 460 whisker series are within 9e-13 s of the 0.005 s grid, with no step shorter than one frame (252,019,206 of 252,170,504 steps are one frame, the rest gaps). Contact starts and stops are on the grid and equal frame_start x 0.005.", "deviations": [ "For each whisker the angle timestamps are read in full; the other six processed series are checked to have the same length and the same first and last time, not read in full." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "sync-gap-and-interpolation", "statement": "Processed whisker timestamps have gaps, for instance during the 133 ms blackout at each trial start, because gaps longer than 4 frames (20 ms) were not interpolated.", "source": "Data Records, NWB file, processing group processed_whisker_position, last paragraph", "quote": "these timestamps will in general contain gaps, for instance during the 133 ms blackout synchronization pulse at the start of each trial. This is because we did not interpolate over gaps longer than 4 frames (20 ms)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "No run of interpolated frames is longer than 4 frames. In every session at least 98.4% of trials have a gap at the trial start within 10% of 133 ms for each of C1 to C3 (C0 at least 90.3%). The gap (missing time) is a median 0.135 s, 27 frames, and 0.135 to 0.140 s on 90% of trials.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "bend-null-interpolated", "statement": "Whisker bend is null on interpolated frames.", "source": "Data Records, processed_whisker_position, Table 3 (bend)", "quote": "This is null on interpolated frames", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "Bend is NaN on all 368,927 interpolated frames and on no non-interpolated frame.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "neural-timestamps", "statement": "The neural data timestamps are regularly spaced in the common video timebase, at a rate near but not exactly 30 kHz.", "source": "Data Records, NWB file, acquisition group extracellular array recording, last paragraph", "quote": "The sampling rate is nearly 30 kHz, but not exactly, because it has been converted to the common video timebase used throughout the NWB file", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "The 44 recordings are stored as a starting time and a rate, so spacing is exactly regular; rates are 29,999.9354 to 30,000.0199 Hz, within 0.0003% of 30 kHz, and none is exactly 30 kHz.", "deviations": [ "The files hold no timestamps array, contrary to Table 2; the rate and regular spacing are taken from the stored rate, which equals (samples minus one) over the span." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "neural-video-alignment", "statement": "The neural and video timebases are related by a linear fit whose residuals at the trial synchronization pulses are never more than one frame (5 ms).", "source": "Technical Validation, Alignment data", "quote": "In contrast the neural and video timebase were synchronized very well by a linear fit. Residuals were never more than a single frame (5 ms)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": "reproduce_rodgers_2022.ipynb", "notes": "The pulse in the neural stream is on an analog input of Open Ephys. The 44 .openephys indexes list 360 ADC and 270 AUX channel files besides the 2,880 CH files, but the dandiset holds only 2,816 CH files (64 per session) and all_channels.events holds only its 1024-byte header. The video side would also need the brightness of every frame of 2 TB of video.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "behavior-clock-drift", "statement": "The behavioral (Arduino) timebase fit the other two timebases poorly, with residuals that varied slowly but predictably over the session, so behavioral events were aligned trial by trial.", "source": "Technical Validation, Alignment data", "quote": "We observed that the behavioral timebase was a poor fit to the other two timebases and that its residuals varied slowly but predictably over the session", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": "reproduce_rodgers_2022.ipynb", "notes": "All times in the NWB files are in the video timebase, and the dandiset's other files are videos, Open Ephys files and Kilosort/Phy outputs; its 44 .log files are all named phy.log, one in each of 44 sorting folders under sourcedata (printed), so they are spike-sorting logs. No Arduino log or behavioral timebase is deposited. The paper does not say where the Arduino times are. The README of the original release on Zenodo (ref. 13, doi:10.5281/zenodo.4743837) lists a per-session behavioral_logfile; asked whether the claim should use it, a reader answered that the paper never says what that release holds, so the claim keeps data dataset and the claims file is unchanged (questions/000231.yaml). The answer changed nothing in the test; the outcome stays blocked, data_missing, as before. The Zenodo release was not read.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "unit-quality-isi", "statement": "Every unit in the dataset passed the objective quality criteria, among them at most 5% of inter-spike intervals below 3 ms and at most 20% of the recording below 5% of the mean firing rate.", "source": "Technical Validation, Spike sorting", "quote": "single units had to pass all of the following objective criteria ... \u22645% of the inter-spike intervals less than 3 ms; \u22641.5% change per minute in spike amplitude; \u226420% of the recording at <5% of the mean firing rate", "statistics": [], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "The ISI part holds for all 999 units (largest fraction of intervals below 3 ms 4.93%). The firing-rate part fails under both readings, so the criterion gives not reproduced; with 60 s bins 6 units exceed 20% (largest 23.3%), with one bin per trial 78 do (largest 40.2%); at a 30% bound, 0 and 6. Leaving out the 16 units of KF134 2018-02-09, whose neural clock the data check did not establish, changes neither count. Amplitude criteria, outside the outcome as the criterion says. 893 of 999 units match their Kilosort cluster by id with equal spike counts, and their NWB spike times equal the Kilosort samples over the rate plus a constant (spread 2.3e-12 s); the other 106 have more Kilosort spikes than NWB spikes. None of the 893 changes amplitude by more than 1.5% per minute (largest 1.47%, median 0.33%). The criteria on amplitudes below the detection threshold and below 10 uV, and on common-mode artefacts, were not run.", "deviations": [ "In reading (a) the last, partial 60 s bin of each unit's interval is left out (a standard choice).", "The change per minute in spike amplitude is measured, for information only, as the slope of a least-squares line of Kilosort amplitude against time over the mean amplitude; the paper does not say how it measured it. Kilosort amplitudes are template scaling factors, not microvolts, and the paper gives no detection threshold, so the other amplitude criteria cannot be run." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "unit-depth-layer", "statement": "Each unit's depth is the depth of the electrode its neuron_channel indexes, and its layer is assigned from that depth.", "source": "Data Records, NWB file, units table, Table 5", "quote": "This is calculated by taking the depth of the electrode in the electrodes table which is indexed by the field \"neuron_channel\" for this unit ... This specifies the laminar location of the unit, based on its depth", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "All 999 units have the depth of the electrodes row their neuron_channel indexes, and electrode ids equal row indexes. The layer depth ranges do not overlap and run 1 (10 to 127 um), 2/3, 4, 5a, 5b, 6, 6b (1372 to 1424 um); the files spell L1 to L6 this way, with sublayers. Each unit's layer is the layer in its electrode's location string for all 999 units once case is ignored; the 446 that differ as spelled are 5a, 5b and 6b against 5A, 5B and 6B.", "deviations": [ "The layer names of the files (\"1\", \"2/3\", \"4\", \"5a\", \"5b\", \"6\", \"6b\") are read as L1 to L6, with 5a and 5b in L5 and 6b in L6." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "electrodes-order", "statement": "The NWB electrodes are ordered from superficial to deep, broken channels are excluded, the raw data columns follow the electrodes table, and the sites are 20 um apart over 1280 um.", "source": "Methods, Ordering of neural data channels; Methods, Neural recordings during behavior, paragraph 4; Data Records, electrodes table", "quote": "we ordered the channels in the Neurodata Without Borders (NWB) file in order of the anatomical location of the corresponding recording site, from superficial to deepest. We excluded broken channels from the NWB file", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "In all 44 sessions y increases strictly in multiples of 20 um (26 sessions have a larger step at a broken channel), the table has 56 to 64 rows, as many as the data columns, and the ElectricalSeries rows are 0 to n-1. All 2,747 location strings hold the five items, and y minus 20 um times the anatomical index including broken channels is constant in each session, so a full probe spans 1260 um between end sites.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "recording-columns", "statement": "Each recording session names the barrel column the probe was in, sampling typically C1, C2 and C3.", "source": "Data Records, electrodes table, Table 4 (electrode_group); Methods, Neural recordings during behavior, paragraphs 2 and 3", "quote": "Its \"location\" field specifies the barrel-related cortical column in which the recording device was inserted during this session", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "Each of the 44 sessions has one electrode group naming a column; C3 13, C1 10, C2 10, D2 3, C4 2, D1 2, and one each of B2, B3, E1 and E2.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "units-obs-intervals", "statement": "Each unit has one observation interval, the start and stop of the neural recording, with its spike times in the common video timebase.", "source": "Data Records, NWB file, units table, Table 5 (obs_intervals, spike_times)", "quote": "In this dataset, there is only one contiguous chunk of data included per session, so the length of this list is one", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "All 999 units have one interval holding all their spikes. It starts at the first sample time and ends one sample period after the last sample time, within the criterion's one sample.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "pose-structure", "statement": "Each session holds pose tracking of four whiskers with eight markers each, whose pixel to millimetre conversion is the same for all markers within a session.", "source": "Data Records, NWB file, processing group raw_whisker_position", "quote": "It comprises four data interfaces, each of which is an ndx_pose.pose.PoseEstimation containing pose tracking data for one of the four whiskers that we tracked. In turn, each PoseEstimation comprises eight PoseEstimationSeries ... which is the same for all markers within a session but can vary across sessions", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "All 115 sessions hold four PoseEstimation objects with eight series of shape (N, 2) each, and one conversion value per session; it varies across sessions from 0.0403 to 0.0524 mm per pixel.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "c0-rare-contacts", "statement": "The straddler whisker C0 rarely made contact with the shapes in most mice.", "source": "Methods, Two behavioral tasks, paragraph 5", "quote": "but this whisker rarely made contact with the shapes in most mice", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "Over trials to analyse, C0 makes fewer contacts than each of C1, C2 and C3 in 10 of 15 mice (C0 fraction 0.0% to 7.3%); in 219CR, 234CR, 245CR, 255CR and KF119 it makes 20% to 49% of contacts.", "deviations": [ "A whisker with no contacts table in a session (C0 in 21 sessions, C3 in 8) is counted as making no contacts there." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "session-min-trials", "statement": "Sessions with fewer than 90 trials to analyse were excluded.", "source": "Technical Validation, Inclusion criteria", "quote": "We also excluded sessions with fewer than 90 trials to analyze", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "methods_underspecified", "notebook": "reproduce_rodgers_2022.ipynb", "notes": "The readings disagree, and the criterion blocks the claim in that case. Under (a), ignore_trial False, one session (228CR, 2019-07-15) has 84 trials; under (b), random and not spoiled, it has 101 and every session has at least 90. Which trials the paper counts is a question for the paper's authors (findings).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "session-labeling-quality", "statement": "Each included session has every whisker labeled in at least 95% of frames, and at most 2% of its contact events contain a frame with a missing label.", "source": "Technical Validation, Inclusion criteria", "quote": "we required that every whisker be labeled in \u226595% of the frames, that \u22642% of the contact events contained even a single frame with a missing label", "statistics": [], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "The contact part fails under every reading; up to 7.7% of a whisker's contacts in a session hold an interpolated or absent frame, and 16 sessions exceed 2% for C1 to C3 (27 for C0 to C3). The frame part holds for C1 to C3 (at least 99.35% of processed frames not interpolated, at least 97.6% of video frames with a pose) but not for C0 (93.1% and 89.7% at worst). Measured values are under whiskers (i) and frames (a). The stop frame belongs to the contact, duration_frames equals frame_stop - frame_start + 1 on all 290,345 contacts and stop_time equals frame_stop x 0.005 s; with stops taken as exclusive the largest fraction is still 7.7%. The expected value is per whisker and session. With contacts pooled over the whiskers of a session (variant), the largest fraction is 2.007% for C0 to C3 and 2.008% for C1 to C3, both in KM91 2016-12-16 (54 of 2,691 and 54 of 2,689 contacts), just above 2%; 1 session (i) or 2 sessions (ii) exceed 2%. It would be 1.94% with exclusive stops, which the files rule out. Pooled, the result rests on one contact (53 of 2,691 would be 1.97%) and on counting a frame absent from the processed series as missing, as the criterion says. Whether the 2% bound is per whisker or pooled over a session's contacts is asked of a reader (questions/000231.yaml); the outcome is tested on the claims file as it is.", "deviations": [ "A contact covers its frames from start to stop inclusive, as duration_frames = frame_stop - frame_start + 1 and stop_time = frame_stop x 0.005 s in the files (shown in the notebook)." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "contacts-structure", "statement": "Contacts are stored per whisker as time intervals with start, stop, location in the frame and whisker angle.", "source": "Data Records, NWB file, processing group identified_whisker_contacts", "quote": "We have also added optional fields containing the location of each contact, expressed both as a Cartesian location within the frame and also as the angle of the whisker at the time of the contact", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "All 431 contacts tables (290,345 rows) have start_time, stop_time, tip_x, tip_y, base_x, base_y and angle, and no row starts after it stops. 29 whisker-sessions have no table (C0 in 21, C3 in 8), read as whiskers that made no contact, as the criterion's wording allows.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "nwb-structure", "statement": "Each NWB file holds the groups Table 1 lists, acquisition (neural recording and behavioral video), electrodes, processing, trials and units, with licks, rewards, raw and processed whisker position and whisker contacts under processing.", "source": "Data Records, Annotated data in the NWB file, Table 1 and the sections after it", "quote": "processing A group containing a ProcessingModule named \"behavior\". This includes processed data about behavioral events, whisker position, and whisker contacts", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "Under the naming of Table 1 and UsageNotes.ipynb (one module \"behavior\"), all 115 files hold left and right licks and rewards, the four raw pose and processed position interfaces with the seven Table 3 series, contacts, the 14 Table 6 columns and a video link that resolves to a deposited video; the 44 ecephys files hold the recording, units with the Table 5 columns, and electrodes with the Table 4 columns (y, location and one electrode group per session, x and z null). The group names of the Data Records sections are used in no file. 29 files hold three contacts tables, not the four the Data Records describe.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "file-layout", "statement": "The dataset has one folder per mouse with one NWB file per session, named sub-_ses-_behavior+image.nwb with \"ecephys\" added when neural data are present, the video of each session in a folder of the same name, and raw neural data under sourcedata.", "source": "Data Records, Directory structure of the entire dataset", "quote": "At the top level, there is one folder per subject ... Video files are stored in a directory with the same name as the corresponding NWB file. Finally, we also provide the raw neural data files in a directory called \"sourcedata\"", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "All 115 NWB files are in their subject's folder and named as described (71 behavior+image, 44 behavior+ecephys+image), each with one video in a folder of its name. sourcedata holds a folder for each of the 44 ecephys sessions and no other, each with Open Ephys files and Kilosort/Phy outputs.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "video-format", "statement": "The raw videos are mkv files whose frame rate is set to 30 frames/s though the true rate is 200 frames/s, and most sessions are 640 x 550 pixels.", "source": "Data Records, Raw data, video files; Methods, High-speed videography and whisker tracking, paragraph 1", "quote": "Importantly, the frame rate of this video file is spuriously set to 30 frames per second, even though the true frame rate is 200 frames per second", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "All 115 videos are Matroska, H.264, at a nominal 30 frames/s; 111 are 640 x 550 and 4 are 640 x 600. Frames over 200 exceed the last whisker time by 0.005 to 0.065 s (1 to 13 frames).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "subjects-sex-age-strain", "statement": "The mice were C57BL/6J, males and females in roughly equal proportion, and began training between postnatal days 90 and 180.", "source": "Methods, Two behavioral tasks, paragraph 7", "quote": "We purchased breeder mice from the C57BL/6 J strain ... Mice began behavioral training between postnatal days 90 and 180 ... We used males and females arbitrarily and in roughly equal proportion", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_rodgers_2022.ipynb", "notes": "All 15 mice are C57BL/6J (spelled C57BL6/J), 10 female and 5 male (binomial p = 0.302, minority one third, so both sex readings hold), 197 to 436 days old at their first session.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "tracking-mean-error", "statement": "The mean distance between true and predicted marker labels is 2.8 pixels.", "source": "Technical Validation, Validation of pose tracking data, paragraph 1", "quote": "The simplest metric of tracking accuracy is the mean distance between the true label and the predicted label, which is 2.8 pixels in our case", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "The human-curated labels and held-out predictions are not in the dandiset, whose files are NWB files, videos and Open Ephys and Kilosort files. The paper reports the errors as reported previously (ref. 8) and names no archive for the curated labeled data the abstract offers. The README of the original release on Zenodo (ref. 13, doi:10.5281/zenodo.4743837) lists a folder trainset/ with per-frame error tables; asked whether the claims should use it, a reader answered that the paper never says what that release holds, so the claims keep data dataset and the claims file is unchanged (questions/000231.yaml). The answer changed nothing in the test; the outcome stays blocked, data_missing, as before. The Zenodo release was not read.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "tracking-error-dataset1", "statement": "On curated dataset 1, 7433 randomly chosen typical frames, each type of tracking error (poorly traced, incorrectly classified, false positive, false negative) is below 0.2%.", "source": "Technical Validation, Validation of pose tracking data, paragraphs 3 and 4; Fig. 2c", "quote": "each type of error is nearly zero (<0.2%) on dataset 1, which is the only dataset representative of typical frames", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "As for tracking-mean-error. The reader's answer (questions/000231.yaml) kept the claim on the dandiset, data dataset, and changed nothing in the test; the outcome stays blocked, data_missing, as before.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "tracking-error-challenge", "statement": "Error rates rise with each later set of challenge frames (curated datasets 2 to 5), and the most common error is the false negative.", "source": "Technical Validation, Validation of pose tracking data, paragraph 4; Fig. 2c", "quote": "Paradoxically, error rates increase with each subsequent set of challenge frames ... The most common type of error is the false negative", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "As for tracking-mean-error; depends on tracking-error-dataset1. The reader's answer (questions/000231.yaml) left the claim unchanged; the outcome stays blocked, data_missing, as before.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "prior-task-strategies", "statement": "In the original study, shape discrimination mice compared the prevalence of contacts across whiskers, a computation enabled by tuning changes in barrel cortex, while shape detection mice responded to the total number of contacts across all whiskers and barrel cortex neurons kept their expected tuning.", "source": "Background & Summary, paragraph 4 (findings of ref. 8)", "quote": "We found that shape discrimination mice compared the prevalence of contacts across whiskers, a comparison computation enabled by tuning changes in the barrel cortex. In contrast, shape detection mice simply responded to the total number of contacts across all whiskers, and in that case barrel cortex neurons maintained their expected tuning", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "methods_underspecified", "notebook": null, "notes": "A finding of ref. 8 restated without a method; the claims file blocks it.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "prior-orthogonal-subspaces", "statement": "In earlier work on these data, barrel cortex represented sensory and motor variables in approximately orthogonal subspaces.", "source": "Background & Summary, paragraph 4 (finding of ref. 10)", "quote": "We have also reported that barrel cortex represents sensory and motor variables in approximately orthogonal subspaces, which could be useful for generalization to novel conditions", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "methods_underspecified", "notebook": null, "notes": "A finding of ref. 10 restated without a method; the claims file blocks it.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null } ], "findings": [ { "kind": "data_missing", "statement": "The analog input that carries the trial synchronization pulse in the neural stream is not deposited. Each session's Continuous_Data.openephys lists 100_ADC1 to 100_ADC8 and 100_AUX1 to 100_AUX6, but sourcedata holds only the 64 100_CH files, and all_channels.events holds no events, so the neural to video alignment cannot be checked from the raw data.", "found": null, "status": null }, { "kind": "data_missing", "statement": "The behavioral (Arduino) timebase is not in the dataset, and neither are the curated whisker training sets and held-out predictions behind the tracking accuracy figures, which the abstract offers to users. The paper does not say where they are; the README of the original release (Zenodo doi:10.5281/zenodo.4743837) lists a trainset/ folder and per-session behavioral log files, which the paper does not mention.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "The keywords of all 30 detection sessions include \"shape discrimination\", contrary to their experiment description (\"whisker-based shape detection\").", "found": null, "status": null }, { "kind": "metadata_error", "statement": "In every file the right reward series (described as the times of reward deliveries to the right) is identical to the right lick series, 315,782 times in all against 12,447 correct right-rewarded trials, so right reward times are not available.", "found": null, "status": null }, { "kind": "metadata_insufficient", "statement": "1,533 trials have stimulus \"flatter convex\" or \"flatter concave\", values that Table 6 does not list or describe.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "choice_time is NaN on all 265 spoiled trials, where Table 6 says it is 45 s after the window opens. On the 22% of the other trials where it is not the first lick in the window (7,185 of 32,085), it falls 0 to 5 ms after the window opening on 7,183, with no lick within 5 ms of it on 94% of them, so it is not the first lick after the window as Table 6 defines it.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "160 spoiled trials have ignore_trial False, where Table 6 says spoiled trials have it True; in the files ignore_trial equals \"non-random or optogenetic\" on every trial.", "found": null, "status": null }, { "kind": "paper_inconsistency", "statement": "The Data Records sections name processing groups behavioral_events, raw_whisker_position, processed_whisker_position and identified_whisker_contacts, with interfaces whisker_C2; the files, Table 1 and the paper's UsageNotes.ipynb use one module behavior with raw_position_whisker_C2, processed_position_whisker_C2 and contacts_by_whisker_C2. The access code printed in the Data Records fails on these files.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "The neural recording has no timestamps array, which Table 2 and the paper's access code (.timestamps[:]) describe; its times are stored as a starting time and a rate.", "found": null, "status": null }, { "kind": "other", "statement": "29 files hold three contacts tables, not the four the Data Records describe (no C0 table in 21, no C3 table in 8), without a note on whether the whisker made no contact.", "found": null, "status": null }, { "kind": "other", "statement": "The dataset does not meet the paper's inclusion criteria as the files allow them to be checked. In 27 sessions more than 2% of a whisker's contacts hold an interpolated or absent frame (16 sessions for C1 to C3, up to 7.7%), C0 is labeled in fewer than 95% of frames in 3 to 7 sessions, and one session (228CR, 2019-07-15) has 84 trials with ignore_trial False.", "found": null, "status": null }, { "kind": "other", "statement": "The paper's 18,514 trials cannot be recovered from the files, which hold 32,350 trials, 22,905 with ignore_trial False; no other combination of the trial flags tried gives it. The video frame total (63,979,800) and the 44 neural sessions match exactly.", "found": null, "status": null }, { "kind": "methods_unavailable", "statement": "Question for the paper's authors: how is the typical 9 s timeout measured? After an error, stop_time minus choice_time is a median 12.21 s, and 8.34 s longer than after a correct trial; only the second is near 9 s.", "found": null, "status": null }, { "kind": "methods_unavailable", "statement": "Question for the paper's authors: which trials count as \"trials to analyze\" in the rule that excludes sessions with fewer than 90? With ignore_trial False one included session has 84; with random, non-spoiled trials every session has at least 90.", "found": null, "status": null } ], "reconciliation": null, "verifications": [ { "date": "2026-10-29", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The handover of the first run, the executed notebook with the conclusions and summary removed, the helper module and the claims file, with the paper under paper/. The verifier also checked the archive's asset list for the blocked claims.", "agreed": 39, "of": 39, "disagreements": [] }, { "date": "2026-10-29", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "Round 2. The full executed notebook of the revised run, with its conclusions and summary, the helper module, the README, the attempt record and the claims file, with the paper under paper/. The verifier agreed with all 39 outcomes and asked for text corrections; the README run time and the Method paragraph of cell 0 were also corrected in reply.", "agreed": 39, "of": 39, "disagreements": [] } ] } ], "entries": [ { "name": "000231-rodgers-2022", "type": "reproduction", "title": "Behavioral, videographic and neural dataset on object recognition in mice (Rodgers 2022)", "description": "Tests the statements of the data descriptor Rodgers (2022, Sci Data 9, 620) on dandiset 000231, the 115 sessions of 15 mice doing whisker-based shape discrimination or detection, with whisker tracking from 200 frames/s video and, in 44 sessions, barrel cortex recordings. It checks the counts, the file layout, the NWB contents, the trial logic, the shared timebase, the unit and electrode tables and the inclusion criteria against the files.", "datasets": [ "dandi:000231" ], "papers": [ "10.1038/s41597-022-01728-1" ], "derived_from": [], "notebooks": [ { "path": "reproduce_rodgers_2022.ipynb", "title": "Reproducing Rodgers (2022) from dandiset 000231", "executed": "notebooks/000231-rodgers-2022/reproduce_rodgers_2022.html" } ] } ] }