{
 "dataset": {
  "archive": "dandi",
  "id": "000129",
  "label": "DANDI:000129"
 },
 "page": "datasets/dandi/000129/",
 "generated": "2026-11-05T07:41:23+00:00",
 "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.",
 "outcome_counts": {
  "reproduced": 10,
  "partially_reproduced": 2,
  "not_reproduced": 6,
  "blocked": 12
 },
 "outcome_counts_with_outside_data": {
  "reproduced": 0,
  "partially_reproduced": 0,
  "not_reproduced": 0,
  "blocked": 0
 },
 "outcome_counts_elsewhere": {
  "reproduced": 0,
  "partially_reproduced": 0,
  "not_reproduced": 0,
  "blocked": 1
 },
 "outcome_counts_links": {
  "reproduced": 0,
  "partially_reproduced": 0,
  "not_reproduced": 0,
  "blocked": 0
 },
 "attempts": [
  {
   "id": "000129-makin-2018-01",
   "dataset": {
    "archive": "dandi",
    "id": "000129",
    "version": "0.241017.1444",
    "doi": "10.48324/dandi.000129/0.241017.1444"
   },
   "key": "dandi:000129",
   "label": "DANDI:000129",
   "dataset_version": "0.241017.1444",
   "paper": "10.1088/1741-2552/aa9e95",
   "paper_title": " Superior arm-movement decoding from cortex with a new, unsupervised-learning algorithm.",
   "date": "2026-10-28",
   "kind": "agent",
   "accountable": "CatalystNeuro",
   "agent": {
    "harness": "Claude Code",
    "model": "claude-opus-5-5",
    "protocol": "pilot-12"
   },
   "runner": {
    "platform": "GitHub Actions",
    "url": "https://github.com/catalystneuro/encore/actions/runs/40749995806"
   },
   "tier": "agent_verified",
   "deprecated": "",
   "deprecated_on": null,
   "entry": "000129-makin-2018",
   "entry_title": "Arm-movement decoding with the rEFH and seven other decoders (Makin et al. 2018)",
   "description": "Tests the findings of Makin, O'Doherty, Cardoso and Sabes (2018) on dandiset 000129, which holds one session of monkey I (M1 array, 649 s of sorted-unit spikes with fingertip and cursor kinematics) from the Neural Latents Benchmark. It re-implements the paper's eight decoders (linear regression, KFobs, KFlatent static and dynamic, Wiener filter, UKF, rEFH static and dynamic) from its Methods and code, and compares them at four bin widths, three amounts of training data and four fractions of the units. The paper's tests across its 49 sessions cannot be run on one session.",
   "notebooks": [
    {
     "path": "reproduce_makin_2018.ipynb",
     "title": "Decoders at 64 ms on dandiset 000129 (figures 2, 4 and 8)",
     "executed": "notebooks/000129-makin-2018/reproduce_makin_2018.html"
    },
    {
     "path": "binwidth_training_neurons.ipynb",
     "title": "Bin width, training data and number of neurons (figures 5 to 7)",
     "executed": "notebooks/000129-makin-2018/binwidth_training_neurons.html"
    }
   ],
   "outcomes": [
    {
     "claim": "units-monkey-i",
     "statement": "After units firing below 0.5 Hz were discarded, monkey I's sessions had between 171 and 413 units, the range reflecting that only 7 of 37 sessions included the S1 array.",
     "source": "Methods, Data acquisition, paragraph 4",
     "quote": "Finally, units with spike rates below 0.5 Hz were discarded. This left between 171 and 413 units for monkey I",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "not_reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "111 of the 130 units in the file fire above 0.5 Hz over the 649.1 s session, below the criterion's range of 153.9 to 454.3. The file holds sorted units only (unit ids end in 1 to 4, none in 00), on 82 channels; the paper's counts include one unsorted hash unit per channel (Methods, Data acquisition), up to 96 more on this array, so the shortfall is consistent with the missing hash units and does not contradict the paper. With thresholds of 0.25 and 1 Hz the count is 117 and 94. No unit is a duplicate of another.",
     "deviations": [
      "One session, the only one in the dataset (dated 2017-02-02, not in the paper's session list), for a range over 37 sessions.",
      "The rate is taken over the training file (649.1 s); the test file holds 190.3 s more of 98 of the units."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig2-refh-kf-best",
     "statement": "Averaged over the 49 sessions, no decoder outperforms the rEFH (+KF) for any kinematic variable, and for velocity and acceleration it is clearly superior to all other decoders.",
     "source": "Results, Reconstruction quality of all decoders, bullet 1; Figure 2",
     "quote": "On average, no decoder outperforms the rEFH (+KF) for any kinematic variable. For velocity and acceleration, the rEFH (+KF) is clearly superior to any of the other decoders.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "The criterion needs the bootstrap of the session-weighted mean SNR difference over the paper's 49 sessions. The dandiset holds one session, which is not in the paper's list. The paper's 47 Sabes-lab sessions are public on Zenodo (doi:10.5281/zenodo.583331), but the claim is on the dataset (`data` is not `elsewhere`), so a reader is asked whether it should be (questions/000129.yaml). On the one session the mean part holds. The rEFH (+KF) has the highest SNR for position, velocity and acceleration (5.91, 5.06 and 2.36 dB), ahead of the next decoder by 0.28 dB (UKF), 1.19 and 0.45 dB (static rEFH), and it is highest with the other seeds and the code's kinematics (position margin 0.11 to 0.51 dB). This does not decide the outcome.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "intro-gain-over-nearest",
     "statement": "The rEFH (+KF) improves SNR over its nearest competitor on average by 0.14 dB for position, 1.60 dB for velocity and 1.2 dB for acceleration; the position gain is not significant at p = 0.01.",
     "source": "Introduction, paragraph 4",
     "quote": "average improvements over its nearest competitor in signal-to-noise ratio of 0.14 dB (position), 1.60 dB (velocity) and 1.2 dB (acceleration) ... the improvement in position is not significant at the level of p = 0.01",
     "statistics": [
      {
       "about": "position gain over the nearest competitor",
       "test": "bootstrap",
       "p": 0.01,
       "p_relation": ">"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "The criterion needs the bootstrap over the 49 sessions for the position gain (p >= 0.01); see fig2-refh-kf-best. On the one session the gains of the rEFH (+KF) are 0.28, 1.24 and 1.02 dB over the nearest decoder other than the two rEFH decoders (reading a; UKF, UKF and WF) and 0.28, 1.24 and 1.25 dB over the UKF (reading b). Position and velocity are outside tolerance under both readings and with every seed (position 0.11 to 0.28 dB, velocity 1.24 to 1.31 dB); with the code's kinematics velocity is inside (1.45 and 1.55 dB) and position is 0.51 dB. No reading gets all three gains within tolerance on this session. This does not decide the outcome.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig2-velocity-boost",
     "statement": "Relative to the static linear map, the rEFH (+KF) raises velocity SNR by more than twice as much as KFobs and nearly twice as much as the UKF.",
     "source": "Results, Reconstruction quality of all decoders, bullet 1; Figure 2, second row",
     "quote": "the average SNR boost provided by the rEFH (+KF) for velocity is more than twice that supplied by KFobs ... nearly twice that of its nearest rival, which is the unscented Kalman filter (UKF)",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "not_reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "Velocity SNR gains over linear regression are 3.67 dB for the rEFH (+KF), 1.91 for KFobs and 2.42 for the UKF. The rEFH's gain is 1.92 times KFobs's (the criterion needs more than 2) and 1.51 times the UKF's (inside 1.5 to 2.2), so only the UKF part holds, which the criterion does not grade as partial. The result is borderline. With the code's kinematics (cursor, no filter, bin means) the ratios are 2.05 and 1.70 and the claim is reproduced; with rEFH seeds 1 and 2 the KFobs ratio is 1.93 and 1.95. The copy the verifier reviewed, from an earlier run of the same notebook, gave 1.97 with seeds 1.92 to 1.99, so the ratio also moves by a few hundredths between runs; every run under the Methods kinematics has been below 2. The outcome rests on the kinematics reading. Question for the paper's authors (finding below) was the decoded kinematic state the fingertip position filtered at 10 Hz and subsampled, as the Methods say, or the unfiltered cursor position averaged over each bin, as getLatentsBMI.m and binSpikeCounts.m do?",
     "deviations": [
      "One session in place of 49; the session-weighted means are this session's values, so a margin this small carries one session's noise.",
      "The kinematics follow the Methods (fingertip, 4th-order Butterworth at 10 Hz, last sample of each bin), as the claims file makes its main reading; the code's reading changes the outcome, and this reading is adopted as the one the paper's text states.",
      "Sorted units only (111), without the hash units the paper used.",
      "The UKF is a re-implementation of Li's UKF2; its fitting code is not in the paper's repository."
     ],
     "assumed": true,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig2-kflatent-dynamic-little-effect",
     "statement": "Adding the dynamic mapping (a second Kalman filter) to KFlatent changes its reconstruction little.",
     "source": "Results, Reconstruction quality of all decoders, bullet 2; Figure 2",
     "quote": "Dynamic mapping (i.e. a KF) of KFlatent has little effect on reconstruction.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "KFlatent (+KF) minus static KFlatent is 0.18, 0.21 and 0.09 dB for position, velocity and acceleration, all below 0.5 dB. With the two other EM seeds it is 0.18 to 0.19, 0.17 to 0.21 and 0.10 to 0.11 dB, and with the code's kinematics 0.29, 0.45 and 0.22 dB. EM ran the full 100 epochs in every run; the last epoch's gain was 4.3e-4 to 6.6e-4 of the total in the six runs of the first notebook (the code-kinematics run 4.3e-4), and 4.3e-4 to 9.4e-4 over every configuration of the second, above the Methods' criterion of 1e-4.",
     "deviations": [
      "One session in place of 49.",
      "Sorted units only (111); EM convergence judged on the log-likelihood, which the Methods' cross-entropy criterion is equivalent to up to a constant."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig2-refh-static-vel-acc",
     "statement": "Under the static mapping alone, velocity and acceleration are reconstructed better from the rEFH than from any other filter.",
     "source": "Results, Reconstruction quality of all decoders, bullet 3; Figure 2",
     "quote": "Under the static mapping alone, velocity and acceleration can be reconstructed better with the rEFH than with any other filter.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "The static rEFH has 3.87 dB velocity SNR against the best other decoder's 3.82 (UKF), and 1.90 dB acceleration SNR against 1.34 (WF). The velocity margin is small, 0.05 dB here, 0.08 and 0.17 dB with rEFH seeds 1 and 2, and 0.09 dB over KFlatent (+KF) with the code's kinematics; the claim holds in all six runs, the EM seeds included.",
     "deviations": [
      "One session in place of 49, so the small velocity margin carries one session's noise.",
      "Sorted units only (111); the UKF is a re-implementation of Li's UKF2."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig2-position-dynamic-gain",
     "statement": "Position is reconstructed much more accurately from the rEFH with the dynamic than with the static mapping, by about 3 dB of SNR.",
     "source": "Results, Reconstruction quality of all decoders, bullet 4; Discussion, Relative reconstruction quality of position, velocity, and acceleration, paragraph 1; Figure 2",
     "quote": "That position, on the other hand, can be reconstructed much more accurately from the rEFH with dynamic, as opposed to static, mappings ... the dynamic mapping (+KF) improves signal-to-noise by some 3 dB",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "not_reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "The dynamic mapping raises the rEFH's position SNR from 1.46 to 5.91 dB, a gain of 4.45 dB (4.32 for x, 4.58 for y), outside 3 +- 0.5 dB. The effect is present and larger than the paper's; it is 4.22 to 4.52 dB with the other seeds and the code's kinematics. The static map has position R2 0.39 and 0.51 (x, y) on the training bins it was fitted on and 0.19 and 0.37 on the test bins, so the second KF, fitted on the training outputs as the paper's dynamic map is, sees outputs less noisy than those it filters at test. Whether this makes the gain larger than the paper's is not tested here.",
     "deviations": [
      "One session in place of 49.",
      "Sorted units only (111)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig2-sagittal-better",
     "statement": "For each decoder and each derivative order, the sagittal (y) component is decoded better than the lateral (x) component.",
     "source": "Results, Reconstruction quality of all decoders, bullet 5; Figure 2",
     "quote": "For each decoder and derivative order (position, velocity, acceleration), variables are decoded better in the sagittal ... than lateral (x) direction",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "partially_reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "y has the higher SNR in 23 of 24 decoder and variable pairs; the exception is the static rEFH's velocity (y minus x -0.06 dB). Linear regression's velocity margin is 0.01 dB. With rEFH seed 2 all 24 hold; with rEFH seed 1, the two other EM seeds and the code's kinematics 23, with the same exception. The copy the verifier reviewed, from an earlier run of the same notebook, gave 24 of 24, so the outcome depends on the training run of the rEFH.",
     "deviations": [
      "One session in place of 49; the per-session comparison is this session's.",
      "Sorted units only (111)."
     ],
     "assumed": false,
     "disagreed": true,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig2-linear-worst",
     "statement": "Static linear regression reconstructs much worse than the other decoders.",
     "source": "Results, Reconstruction quality of all decoders, paragraph 1; Figure 2",
     "quote": "improvement in reconstruction over linear regression, which is much worse than the other six decoders",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "Every other decoder exceeds linear regression for each of the six variables; the smallest margin is 0.45 dB (static rEFH, x position). It holds in every variant (smallest 0.20 dB, KFobs x acceleration, code kinematics).",
     "deviations": [
      "One session in place of 49.",
      "Sorted units only (111)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig2-gross-failures",
     "statement": "Gross failures (negative coefficients of determination) occur only for the UKF and KFlatent (+KF), in a few sessions.",
     "source": "Results, paragraph after the decoder list",
     "quote": "in our experiments, such gross failures occurred only for the UKF and KFlatent (+KF)",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "No decoder has a negative R2 at 64 ms in this session (the lowest is 0.055, linear regression), so the criterion holds trivially; the claim's \"in a few sessions\" cannot be tested on one session. Outside the criterion's 64 ms with all units and 320 s of training, the second notebook prints negative R2 for linear regression (16 ms, 80 and 160 s, f = 1/8 and 1/4), static KFlatent, KFlatent (+KF) and the static rEFH (f = 1/8 and 1/4), KFobs (f = 1/4, x position), the UKF (f = 1/8, ay, and f = 1/4, x position), and KFlatent (+KF) at 16 ms; most are within 0.05 of zero, and at f = 1/8 KFlatent (+KF) diverges (position SNR -76.63 dB). Under a reading of \"in our experiments\" that covers every configuration the claim would fail; a reader is asked (questions/000129.yaml).",
     "deviations": [
      "One session in place of 49.",
      "Sorted units only (111)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig4-vel-acc-every-session",
     "statement": "For velocity and acceleration, the rEFH (+KF) is better than KFobs and the UKF in every one of the 196 session, variable and axis combinations, and significantly so across sessions.",
     "source": "Results, The rEFH against its nearest rivals, all session and monkeys, paragraph 2; Figure 4, columns 2 and 3",
     "quote": "For velocity and acceleration, the rEFH is superior to KFobs and the UKF for all 196 data ... For both kinematic variables, the results are significant under a bootstrap (p < 0.01",
     "statistics": [
      {
       "test": "bootstrap",
       "p": 0.01,
       "p_relation": "<",
       "n": "49 sessions from three monkeys"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "In this session the rEFH (+KF) beats KFobs and the UKF in all 8 velocity and acceleration comparisons, by 1.04 to 1.81 dB, and in every variant (smallest margin 1.04 dB). The criterion's second part, the bootstrap across sessions at p < 0.01, cannot be run on one session; the outcome rests on the per-session part, which the claims file names as testable on one session. The verifier graded it partially reproduced in round 1 and agreed in round 2.",
     "deviations": [
      "One session in place of 49; the bootstrap part of the criterion is not tested.",
      "Sorted units only (111); the UKF is a re-implementation of Li's UKF2."
     ],
     "assumed": false,
     "disagreed": true,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig4-advantage-grows",
     "statement": "The better decoded the session, the more the rEFH (+KF) outperforms the standard Kalman filter, shown by regression slopes above 1 for velocity and acceleration.",
     "source": "Results, The rEFH against its nearest rivals, all session and monkeys, paragraph 2; Figure 4, columns 2 and 3",
     "quote": "the better (more decodable) the session, the more the rEFH outperforms the standard Kalman filter ... confirmed with an F-test against a model with slope set to unity, intercept fit to data",
     "statistics": [
      {
       "test": "F-test against a unit-slope model",
       "p": 0.01,
       "p_relation": "<"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": null,
     "notes": "The regression of rEFH (+KF) SNR on KFobs SNR runs over all sessions and both axes; one session gives two points and no degrees of freedom for the F-test. The dandiset holds no other session; see fig2-refh-kf-best.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig4-position",
     "statement": "The rEFH (+KF) decodes position better than KFobs, but not distinguishably from the UKF.",
     "source": "Results, The rEFH against its nearest rivals, all session and monkeys, paragraph 3; Figure 4, column 1",
     "quote": "The rEFH\u2019s position reconstruction is likewise superior to KFobs (p < 0.01, bootstrap). It is not, however, distinguishable from either the UKF (bootstrap not significant even at p = 0.05).",
     "statistics": [
      {
       "about": "rEFH (+KF) against KFobs, position",
       "test": "bootstrap",
       "p": 0.01,
       "p_relation": "<"
      },
      {
       "about": "rEFH (+KF) against UKF, position",
       "test": "bootstrap",
       "p": 0.05,
       "p_relation": ">"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "Needs the bootstrap over the 49 sessions; see fig2-refh-kf-best. On the one session the rEFH (+KF) position SNR exceeds KFobs's by 0.90 dB and the UKF's by 0.28 dB (0.73 to 1.02 and 0.11 to 0.51 dB across the variants), which does not decide the outcome.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig4-best-variable",
     "statement": "The best-decoded kinematic variable is velocity for the rEFH (+KF), but position for KFobs and the UKF.",
     "source": "Results, The rEFH against its nearest rivals, all session and monkeys, paragraph 3; Figure 4",
     "quote": "the best-decoded kinematic variable is velocity for the rEFH, but position for KFobs and the UKF",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "not_reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "Position is the best-decoded variable for all three decoders. The rEFH (+KF) has 5.91 dB for position against 5.06 for velocity and 2.36 for acceleration; KFobs (5.00, 3.30, 1.02) and the UKF (5.63, 3.82, 1.11) have position highest, as the paper says. The ordering is the same with the other seeds and the code's kinematics, and at every bin width in the second notebook.",
     "deviations": [
      "One session in place of 49.",
      "Sorted units only (111)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig4-task-similar",
     "statement": "The relative advantage of the rEFH (+KF) does not differ visibly between the center-out task (monkey C) and the point-to-point task (monkeys I and L); monkey L's sessions differ most.",
     "source": "Results, Reaching task, paragraph 1; Figure 4",
     "quote": "But across tasks, the relative advantage of the rEFH (+KF) does not differ visibly (there are too few sessions from monkey C for a statistical test)",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "other",
     "notebook": null,
     "notes": "The comparison is of monkey C's two sessions with the range of monkey I's 37 sessions. The dandiset holds one monkey I session, not among the 37. Monkey C's data are named only by the preprint (Dyer et al., biorxiv 080861), with no archive identifier to pin, so they cannot be used as an input (author, step 1.4). The claim is `data elsewhere`, so it is not blocked as data_missing, which the verifier gave.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "elsewhere",
     "where": "Monkey C data, Dyer et al. 2016, biorxiv.org/content/early/2016/10/14/080861 (reference 6); no archive identifier is given in the paper",
     "proposed": null
    },
    {
     "claim": "fig5-vel-acc-all-bins",
     "statement": "For velocity and acceleration, the rEFH (+KF) is the best decoder at every bin width (16, 32, 64 and 128 ms).",
     "source": "Results, Bin width, bullet 1; Figure 5",
     "quote": "For velocity and acceleration, the rEFH (+KF) is the best for all bin widths",
     "statistics": [
      {
       "test": "bootstrap",
       "p": 0.01,
       "p_relation": "<"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": "binwidth_training_neurons.ipynb",
     "notes": "Needs the bootstrap over the 49 sessions at four bin widths; see fig2-refh-kf-best. On the one session the rEFH (+KF) has the highest velocity and acceleration SNR at 32, 64 and 128 ms (margins 0.53 and 0.32, 1.19 and 0.45, 0.45 and 0.12 dB), but at 16 ms the WF is higher for velocity (3.30 against 3.01 dB) and acceleration (1.39 against 0.96 dB), against the claim. This does not decide the outcome.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig5-position-bins",
     "statement": "For position the leading decoders cannot be told apart at any bin width except 16 ms, where the UKF is better.",
     "source": "Results, Bin width, bullet 1; Figure 5",
     "quote": "For position, the leading decoders are not statistically differentiable at any bin width ... except at 16 ms, where the UKF is superior (bootstrap, p < 0.01)",
     "statistics": [
      {
       "about": "UKF against rEFH (+KF), position, 16 ms bins",
       "test": "bootstrap",
       "p": 0.01,
       "p_relation": "<"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": "binwidth_training_neurons.ipynb",
     "notes": "Needs the pairwise bootstrap over the 49 sessions at four bin widths; see fig2-refh-kf-best. On the one session the UKF's position SNR exceeds the rEFH (+KF)'s by 0.29 dB at 16 ms, the direction the paper reports; the absence of differences at 32 to 128 ms can be judged only with the test.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig5-advantage-with-binwidth",
     "statement": "Reconstruction improves with bin width, and so does the relative advantage of the rEFH (+KF).",
     "source": "Results, Bin width, bullet 2; Figure 5",
     "quote": "With increasing bin width, reconstruction improves, as expected; but so does the relative advantage of the rEFH (+KF).",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "binwidth_training_neurons.ipynb",
     "notes": "The rEFH (+KF) SNR rises from 3.01 to 5.46 dB (velocity) and 0.96 to 2.39 dB (acceleration) from 16 to 128 ms, with no fall between neighbouring widths. Its advantage over the best decoder other than the two rEFH decoders rises from -0.29 to 1.60 dB (velocity) and -0.44 to 1.14 dB (acceleration), and in percent from -8.9 to 41.6% and -31.3 to 92.0%, so the claim holds under both readings. The WF's velocity SNR does not improve with width (3.30 to 3.17 dB); the criterion tests only the rEFH (+KF).",
     "deviations": [
      "One session in place of 49.",
      "Sorted units only (111); the UKF is a re-implementation of Li's UKF2."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig5-derivative-order",
     "statement": "With increasing derivative order reconstruction quality declines (except that the rEFH is best at velocity), while the relative advantage of the rEFH (+KF) increases.",
     "source": "Results, Bin width, bullet 3; Figure 5",
     "quote": "With increasing derivative order (position, velocity, acceleration), the overall reconstruction quality declines (except for the rEFH, which is best at velocity), but the relative advantage of the rEFH (+KF) increases.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "partially_reproduced",
     "blocked_by": null,
     "notebook": "binwidth_training_neurons.ipynb",
     "notes": "The mean SNR of the six non-rEFH decoders falls from position to velocity to acceleration at all four widths (64 ms, 3.79, 3.06 and 0.99 dB). The percent advantage of the rEFH (+KF) rises from position to velocity to acceleration at 32, 64 and 128 ms (64 ms, 4.9, 32.6 and 76.5%) but not at 16 ms, where it is -6.2, -8.9 and -31.3%. Both parts hold at 3 of 4 widths, which the criterion grades as partial. The statement's exception (the rEFH best at velocity), which the criterion does not test, does not hold; the rEFH (+KF) decodes position best at every width.",
     "deviations": [
      "One session in place of 49.",
      "Sorted units only (111)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig6-training-time",
     "statement": "The rEFH (+KF) is the best decoder of velocity and acceleration for 80, 160 and 320 s of training data; for position it never differs significantly from the UKF, and beats KFobs at 320 s but loses to it at 80 s.",
     "source": "Results, Seconds of data used for training; Figure 6",
     "quote": "For velocity and acceleration, the rEFH (+KF) is again the best decoder for all amounts of training data. For position, the differences between the rEFH (+KF) and the UKF are nowhere significant, but compared to KFobs, the rEFH is superior at 320 s and inferior at 80 s (bootstrap, p = 0.01)",
     "statistics": [
      {
       "about": "rEFH (+KF) against KFobs, position, 320 s and 80 s",
       "test": "bootstrap",
       "p": 0.01
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": "binwidth_training_neurons.ipynb",
     "notes": "Part (ii) needs the bootstrap over the 49 sessions, and part (iii) its p-values; see fig2-refh-kf-best. On the one session part (i) holds on point values, the rEFH (+KF) having the highest velocity and acceleration SNR at 80, 160 and 320 s (velocity margins 0.33, 1.01 and 1.19 dB). For position it is 0.87 dB below the UKF at 80 s and 0.10 and 0.28 dB above at 160 and 320 s; against KFobs it is above at 320 s (0.90 dB) and below at 80 s (0.20 dB), the directions of part (iii). This does not decide the outcome.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig7-vel-acc-neurons",
     "statement": "For velocity and acceleration, using a fraction 1/4, 1/2 or all of the neurons, the rEFH (+KF) outperforms all its rivals, and its percent gain in SNR grows with the number of neurons and with derivative order.",
     "source": "Results, Number of neurons, paragraph 2; Figure 7",
     "quote": "For velocity and acceleration, for all but the smallest fraction of neurons, the rEFH (+KF) outperforms all its rivals (bootstrap; the result holds for all comparisons at p < 0.01 except against the WF in acceleration at f = 1/4, where it holds for p < 0.05). The percent increase in SNR provided by the rEFH increases with neurons as well as derivative order.",
     "statistics": [
      {
       "about": "all comparisons except WF, acceleration, f = 1/4",
       "test": "bootstrap",
       "p": 0.01,
       "p_relation": "<"
      },
      {
       "about": "against WF, acceleration, f = 1/4",
       "test": "bootstrap",
       "p": 0.05,
       "p_relation": "<"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": "binwidth_training_neurons.ipynb",
     "notes": "Needs the bootstrap over the 49 sessions at each fraction; see fig2-refh-kf-best. On the one session the rEFH (+KF) ties the WF for velocity at f = 1/4 (1.56 dB each) and is the best decoder at f = 1/2 and 1. Its percent gain over the best decoder other than the two rEFH decoders rises with f for velocity (0.2, 19.1 and 32.6%) but not for acceleration (38.2, 87.1 and 76.5%), and over each decoder it also falls somewhere (over the UKF for velocity, 19.4 to 19.1% from f = 1/4 to 1/2). This does not decide the outcome.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig7-position-neurons",
     "statement": "For position the rEFH's advantage is significant at p < 0.05 at f = 1/4 and 1/2, but not at f = 1, where the UKF comes close, nor at f = 1/8, where all decoders collapse.",
     "source": "Results, Number of neurons, paragraph 2; Figure 7, left panel",
     "quote": "Even for position, the rEFH\u2019s advantage is significant at p < 0.05, except for f = 1, where UKF performance gets close, and at the smallest fraction of neurons (f = 1/8)",
     "statistics": [
      {
       "test": "bootstrap",
       "p": 0.05,
       "p_relation": "<"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": "binwidth_training_neurons.ipynb",
     "notes": "Needs the bootstrap over the 49 sessions at each fraction; see fig2-refh-kf-best. On the one session the rEFH (+KF) has the highest position SNR at f = 1/4 (by 0.21 dB) and is 0.15 dB below the UKF at f = 1/2; the copy the verifier reviewed, from an earlier run, had it highest at both. This does not decide the outcome.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig7-affine-position",
     "statement": "Position SNR of the decoders without a learned kinematic model (linear regression, static KFlatent, WF, static rEFH) is close to an affine function of the fraction of neurons, with correlations 0.999, 0.95, 0.98 and 1.000; static KFlatent is the most curved.",
     "source": "Results, Number of neurons, paragraph 3; Figure 7, left panel",
     "quote": "SNRs for the decoders that do not learn the kinematics explicitly ... are quite close to affine functions (i.e. linear with offset) of the fraction of neurons used (\u03c1 = 0.999, 0.95, 0.98, 1.000, respectively). SNR for KFlatent is clearly the most curved of these functions.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "not_reproduced",
     "blocked_by": null,
     "notebook": "binwidth_training_neurons.ipynb",
     "notes": "Correlations of position SNR with f are 0.995 (linear regression), 0.995 (static KFlatent), 0.986 (WF) and 0.991 (static rEFH) under reading a, and 0.956, 0.948, 0.954 and 0.914 with log2 f (reading b); all are within tolerance. Static KFlatent does not have the lowest correlation under either reading (the WF has under a, the static rEFH under b), so the ordering condition fails under both. At f = 1/8 and 1/4 (13 and 27 units) the position SNR of these decoders is within 0.33 dB of zero, so the correlations rest mostly on f = 1/2 and 1.",
     "deviations": [
      "One session in place of 49; fractions of 111 sorted units (13, 27, 55 and 111)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig7-kinematic-model-saturates",
     "statement": "Position SNR of the four decoders that learn a kinematic state-transition model (KFobs, KFlatent (+KF), UKF, rEFH (+KF)) rises much faster at first with added neurons, then saturates.",
     "source": "Results, Number of neurons, paragraph 3; Figure 7, left panel",
     "quote": "In contrast, reconstruction quality for the four decoders (KFobs, KFlatent [+KF], UKF, rEFH [+KF]) that did learn a model of kinematic evolution ... initially grows much more quickly with the addition of neurons, before saturating.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "binwidth_training_neurons.ipynb",
     "notes": "Under reading b (1/8 to 1/2) the position SNR of KFobs, KFlatent (+KF), the UKF and the rEFH (+KF) rises 6.67, 207.33, 6.93 and 6.22 dB per unit f, faster than from 1/2 to 1 (3.79, 6.27, 4.95, 5.79) and faster than the largest initial slope of the static decoders (5.00, static KFlatent). Under reading a (1/8 to 1/4) it fails for KFobs, the UKF and the rEFH (+KF), whose position SNR is slightly lower at 1/4 than at 1/8. KFlatent (+KF) passes under b only because its filter diverges at f = 1/8 (-76.63 dB, 13 units); its slope from 1/4 to 1/2 is 4.20, below its late slope of 6.27, and it would pass only with an f = 1/8 SNR below -1.23 dB, where every other decoder is between -0.10 and 0.68 dB. Without that value the claim fails under both readings, so the outcome rests on a gross failure of one of the four decoders. The other three pass under b on their own slopes.",
     "deviations": [
      "One session in place of 49; fractions of 111 sorted units."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig8-velocity-variants",
     "statement": "Under the static mapping, about 0.7 dB of the rEFH's velocity advantage over the UKF is lost when the updated visible units are left out of decoding, and 0.4 dB when standard-normal visible units replace Poisson ones; Kalman filtering does not remove these differences.",
     "source": "Results, Variant decoders for the rEFH, paragraph 3; Figure 8",
     "quote": "Under a static mapping, some (about 0.7 dB) of that velocity-decoding advantage is lost when the updated visible units are neglected, and a little less (0.4 dB) lost when standard-normal units are used ... The differential between velocity decoders cannot be eliminated by Kalman filtering",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "not_reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "Static velocity SNR is 0.11 dB lower without the updated visible units (text 0.7, figure 8 about 0.3) and 0.08 dB higher, not lower, with standard-normal units on standardized counts (text 0.4). With the code's transform 2 sqrt(R + 3/8) the standard-normal rEFH loses 2.11 dB. Under the dynamic mapping the canonical rEFH (+KF) exceeds the variants by 0.15, 0.15 and 1.68 dB, so that part holds. The static rEFH's own velocity advantage over the UKF is 0.05 dB in this session, against about 0.94 dB in figure 8. Neither standard-normal reading matches.",
     "deviations": [
      "One session in place of 49.",
      "Sorted units only (111).",
      "The standard-normal rEFH uses the code's learning rate for such units (mw0 = 200, setParams.m)."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig8-position-variants",
     "statement": "For position the rEFH variants change the static but not the dynamic mapping; with the Kalman filter all three variants decode position about equally.",
     "source": "Results, Variant decoders for the rEFH, paragraph 2; Figure 8, first panel",
     "quote": "For position, the variants affect the static, but not the dynamic, mappings.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "The dynamic part needs the bootstrap over the 49 sessions; see fig2-refh-kf-best. On the one session the static part holds, the canonical static rEFH having a higher position SNR (1.46 dB) than both variants (1.43 and 1.38). Under the dynamic mapping the canonical rEFH (+KF) is 0.11 and 0.16 dB above them. This does not decide the outcome.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig8-gaussian-still-beats-ukf",
     "statement": "The rEFH keeps an advantage over the UKF when its visible units are standard normal instead of Poisson, so the noise model accounts for only part of its advantage.",
     "source": "Discussion, The importance of nonlinear tuning, Poisson emissions, and a latent state, paragraph 1; Figure 8, medium-gray bars",
     "quote": "the rEFH\u2019s advantage over the UKF is not abolished by abandoning the final \u2018correction\u2019, sc., Poisson rather than normally distributed spike counts (figure 8, medium-gray bars)",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "Trained on standardized counts, the standard-normal rEFH (+KF) exceeds the UKF by 1.09 dB for velocity and 1.17 dB for acceleration. Trained on the code's transform it is 0.43 dB below the UKF for velocity and 0.12 dB above for acceleration, so the claim fails under the code's reading. Question for the paper's authors (finding below) were the standard-normal visible units fed standardized counts, as the Results say, or 2 sqrt(R + 3/8), as getLatentsBMI.m does?",
     "deviations": [
      "One session in place of 49.",
      "The input of the standard-normal rEFH is the standardized counts the Results name (z-scored with the training mean and SD; Results, Variant decoders, \"first standardizing the data (separately for each neuron)\"), as the claims file makes its main reading; the code's transform changes the outcome.",
      "The UKF is a re-implementation of Li's UKF2."
     ],
     "assumed": true,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "fig9-taps",
     "statement": "Adding taps improves both the rEFH (+KF) and the UKF; at 32, 64 and 128 ms the best rEFH decoder beats the best UKF at velocity and acceleration and is indistinguishable at position, while at 16 ms the UKF is better at position and velocity.",
     "source": "Results, Augmenting the decoders with taps, paragraph 2; Figure 9",
     "quote": "this introduction of taps improves the performance of the rEFH (figure 9), as it does the UKF. As before, at all but the smallest bin width, the best rEFH decoder is superior to the best UKF at velocity and acceleration reconstruction, and indistinguishable at position reconstruction (bootstrap; p < 0.01). At that smallest bin width (16 ms), however, the UKF is superior in both position and velocity decoding.",
     "statistics": [
      {
       "test": "bootstrap",
       "p": 0.01,
       "p_relation": "<"
      }
     ],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": null,
     "notes": "Every grade of the criterion needs part (ii), the bootstrap over the 49 sessions at three bin widths, which the dandiset does not hold; see fig2-refh-kf-best. Parts (i) and (iii) are single-session comparisons that could be computed, but whatever they give the claim stays blocked, and the tapped decoders were not implemented. The tapped UKF follows Li's UKF2 (past and future kinematic states in the state, one bin of spike history), whose code is not in the paper's repository, and the tapped rEFH feeds the hidden units to that code without tuning nonlinearities (Results, Augmenting the decoders with taps), so both would be further re-implementations.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "kflatent-modest",
     "statement": "KFlatent is only modestly better than the standard Kalman filter; nonlinear tuning (the UKF) improves on the standard filter more.",
     "source": "Discussion, The importance of nonlinear tuning, Poisson emissions, and a latent state, paragraph 1; Figures 2 and 5 to 7",
     "quote": "yet its performance is only modestly superior to that of the standard Kalman filter, KFobs ... What does improve the standard KF is the addition of tuning to nonlinear and tapped functions of the kinematic state",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "Reading a (velocity), the better KFlatent decoder exceeds KFobs by 0.10 dB and the UKF by 0.51 dB; reading b (velocity and acceleration) 0.08 and 0.30 dB; reading c (all three) KFlatent is 0.20 dB below KFobs. With the two other EM seeds KFlatent leads KFobs by 0.08 and 0.08 dB under reading a and 0.07 and 0.08 dB under reading b, so a and b hold in all three EM runs, though by small margins. With the code's kinematics readings a and b fail (0.49 and 0.39 dB for KFlatent against 0.38 and 0.22 for the UKF) and reading c holds (0.07 against 0.32 dB), so the claim holds under one reading with either kinematics. EM stopped at 100 epochs in every run, not at the Methods' criterion.",
     "deviations": [
      "One session in place of 49.",
      "Sorted units only (111); the UKF is a re-implementation of Li's UKF2."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "advantage-grows-with-derivative",
     "statement": "At 64 ms and with each amount of training data, the relative advantage of the rEFH (+KF) over the other decoders increases from position to velocity to acceleration.",
     "source": "Results, The rEFH against its nearest rivals, all session and monkeys, paragraph 3; Results, Seconds of data used for training; Figures 2, 4 and 6",
     "quote": "Overall, the relative advantage provided by the rEFH increases with derivatives.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "reproduced",
     "blocked_by": null,
     "notebook": "binwidth_training_neurons.ipynb",
     "notes": "The percent advantage of the rEFH (+KF) over the best non-rEFH decoder rises from position to velocity to acceleration with 80 s (-26.1, 10.9, 76.4%), 160 s (2.4, 32.5, 71.3%) and 320 s (4.9, 32.6, 76.5%) of training. It also rises at each amount when the shorter trainings are scored only on the bins after 320 s.",
     "deviations": [
      "One session in place of 49.",
      "For 80 and 160 s of training the paper does not say which bins are tested; the code tests on every bin after the training bins (getLatentsBMI.m, getTestBins), and this is the main reading. Scoring on the bins after 320 s gives the same outcome."
     ],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    },
    {
     "claim": "static-refh-position-worse",
     "statement": "Under the static mapping alone the rEFH reconstructs position significantly worse than the UKF, while the rEFH (+KF) is not distinguishable from it.",
     "source": "Discussion, Relative reconstruction quality of position, velocity, and acceleration, paragraph 1; Figure 8, first panel",
     "quote": "Despite superior velocity and acceleration decoding, the rEFH (+KF) reconstructs position little better than the UKF (the differences are not significant); and under a static mapping, significantly worse.",
     "statistics": [],
     "kind": "finding",
     "inferred": "",
     "outcome": "blocked",
     "blocked_by": "data_missing",
     "notebook": "reproduce_makin_2018.ipynb",
     "notes": "Needs the bootstrap over the 49 sessions, and depends on fig4-position, which is blocked; see fig2-refh-kf-best. On the one session the static rEFH's position SNR is 4.17 dB below the UKF's (1.46 against 5.63; 4.01 to 4.12 dB in the variants), the direction the claim states, which does not decide the outcome.",
     "deviations": [],
     "assumed": false,
     "disagreed": false,
     "data": "dataset",
     "where": null,
     "proposed": null
    }
   ],
   "findings": [
    {
     "kind": "data_missing",
     "statement": "The dandiset holds one session of one of the paper's three monkeys (monkey I, M1 array, 649.1 s with kinematics), released for the Neural Latents Benchmark. The paper used 49 sessions. The file dates the session 2017-02-02, which is not in the paper's list of monkey I sessions (NeuralAnalysis/Indy_HC_Sessions.csv, ending 2017-01-31), so it is probably not one of the paper's sessions, though the dandiset names the paper as describing it. The paper's own sessions are on Zenodo (doi:10.5281/zenodo.583331). The paper's tests across sessions cannot be run on the dandiset.",
     "found": null,
     "status": null
    },
    {
     "kind": "data_missing",
     "statement": "The units table holds sorted units only (ids are channel times 100 plus 1 to 4). The paper kept an unsorted hash unit for every channel and counted them among its 171 to 413 units per monkey I session; without them 111 units pass the 0.5 Hz cut.",
     "found": null,
     "status": null
    },
    {
     "kind": "selection_undocumented",
     "statement": "The training file has three gaps of 0.1 s (162.2 to 162.3, 324.5 to 324.6 and 486.8 to 486.9 s) where every behavioural series is NaN and no unit has a spike, splitting it into four segments; neither the file nor the dandiset says what was removed. The test file (190.3 s, 98 units) has no kinematics, so it cannot be used to score decoding.",
     "found": null,
     "status": null
    },
    {
     "kind": "code_mismatch",
     "statement": "The Methods decode the fingertip position low-pass filtered at 10 Hz and subsampled per bin; the paper's code (getLatentsBMI.m, binSpikeCounts.m) decodes the cursor position with the filter commented out and averages each bin. The two readings change the outcome of fig2-velocity-boost. Question for the paper's authors, which kinematics were decoded?",
     "found": null,
     "status": null
    },
    {
     "kind": "code_mismatch",
     "statement": "The Results say the standard-normal rEFH variant was trained on standardized spike counts; the code (getLatentsBMI.m) feeds it 2 sqrt(R + 3/8). The two readings change the outcome of fig8-gaussian-still-beats-ukf. Question for the paper's authors, which input was used for figure 8?",
     "found": null,
     "status": null
    },
    {
     "kind": "methods_unavailable",
     "statement": "The UKF follows Li's UKF2 package (fit_ar_ukf_hist_rrcv.m, ar_ukf_hist.m), which the paper's repository calls but does not hold; the attempt re-implemented it from the paper's description (linear state transition, tuning to the six states, their magnitudes and the position-velocity products, regression with full noise covariances, standard unscented transform). Question for the paper's authors, can the UKF2 code and its ridge setting be published with the paper's code?",
     "found": null,
     "status": null
    }
   ],
   "reconciliation": null,
   "verifications": [
    {
     "date": "2026-10-28",
     "kind": "agent",
     "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebooks)",
     "reviewed": "The handover of the author-answers run, the executed notebooks with the conclusions and summaries removed, the helper module and the claims file. The verifier also read the paper and queried the DANDI API for 000129 at version 0.241017.1444.",
     "agreed": 29,
     "of": 31,
     "disagreements": [
      {
       "claim": "fig2-sagittal-better",
       "outcome": "reproduced",
       "reason": "y beats x in 24 of 24 pairs; one rEFH seed gives 23. The static rEFH velocity and linear velocity margins are only 0.01 to 0.04 dB."
      },
      {
       "claim": "fig4-vel-acc-every-session",
       "outcome": "partially_reproduced",
       "reason": "The per-session part holds; 8 of 8 comparisons favour the rEFH (+KF), smallest margin 1.08 dB across seeds. The across-session bootstrap cannot be tested (one session)."
      }
     ]
    },
    {
     "date": "2026-10-29",
     "kind": "agent",
     "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebooks)",
     "reviewed": "The full executed notebooks of the author-revise-1 run with their conclusions and summaries, the README, the attempt record, the helper module and the claims file. The verifier also read the paper. It did not check the cited rbmish code details, which were not in its materials, and it notes that the UKF re-implements Li's UKF2, which cannot be checked against the original.",
     "agreed": 31,
     "of": 31,
     "disagreements": []
    }
   ]
  }
 ],
 "entries": [
  {
   "name": "000129-makin-2018",
   "type": "reproduction",
   "title": "Arm-movement decoding with the rEFH and seven other decoders (Makin et al. 2018)",
   "description": "Tests the findings of Makin, O'Doherty, Cardoso and Sabes (2018) on dandiset 000129, which holds one session of monkey I (M1 array, 649 s of sorted-unit spikes with fingertip and cursor kinematics) from the Neural Latents Benchmark. It re-implements the paper's eight decoders (linear regression, KFobs, KFlatent static and dynamic, Wiener filter, UKF, rEFH static and dynamic) from its Methods and code, and compares them at four bin widths, three amounts of training data and four fractions of the units. The paper's tests across its 49 sessions cannot be run on one session.",
   "datasets": [
    "dandi:000129"
   ],
   "papers": [
    "10.1088/1741-2552/aa9e95"
   ],
   "derived_from": [],
   "notebooks": [
    {
     "path": "reproduce_makin_2018.ipynb",
     "title": "Decoders at 64 ms on dandiset 000129 (figures 2, 4 and 8)",
     "executed": "notebooks/000129-makin-2018/reproduce_makin_2018.html"
    },
    {
     "path": "binwidth_training_neurons.ipynb",
     "title": "Bin width, training data and number of neurons (figures 5 to 7)",
     "executed": "notebooks/000129-makin-2018/binwidth_training_neurons.html"
    }
   ]
  }
 ]
}