{ "dataset": { "archive": "figshare", "id": "24949470", "label": "FIGSHARE:24949470" }, "page": "datasets/figshare/24949470/", "generated": "2026-11-05T06:30:02+00:00", "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.", "outcome_counts": { "reproduced": 12, "partially_reproduced": 4, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_with_outside_data": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_elsewhere": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 2 }, "outcome_counts_links": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "attempts": [ { "id": "figshare-24949470-mintal-2024-01", "dataset": { "archive": "figshare", "id": "24949470", "version": "1", "doi": "10.6084/m9.figshare.24949470.v1" }, "key": "figshare:24949470", "label": "FIGSHARE:24949470", "dataset_version": "1", "paper": "10.1038/s41597-024-03777-0", "paper_title": null, "date": "2026-10-27", "kind": "agent", "accountable": "CatalystNeuro", "agent": { "harness": "Claude Code", "model": "claude-opus-5-5", "protocol": "pilot-12" }, "runner": { "platform": "GitHub Actions", "url": "https://github.com/catalystneuro/encore/actions/runs/40572313729" }, "tier": "agent_verified", "deprecated": "", "deprecated_on": null, "entry": "figshare-24949470-mintal-2024", "entry_title": "The Volebny Kompas datasets on Slovak voter and party positions (Mintal et al. 2024)", "description": "Tests the findings of Mintal et al. (2024) on Figshare item 24949470, version 1, which holds 134,699 responses to the Volebny Kompas voting advice application before the 2023 Slovak election, the positions of 11 parties on its 39 statements, the paper's R code and Supplementary Table 1. It checks the counts and structure of both datasets, the coders' agreement, the platform's placement of the parties and their clustering, the demographic table, the summary statistics and the factor analysis.", "notebooks": [ { "path": "reproduce_mintal_2024.ipynb", "title": "Reproducing Mintal et al. (2024) from Figshare item 24949470", "executed": "notebooks/figshare-24949470-mintal-2024/reproduce_mintal_2024.html" } ], "outcomes": [ { "claim": "user-responses-count", "statement": "The User Responses Dataset holds 134,699 voter responses, each with answers to the 39 statements, along with demographics, the three most salient statements and the parties the respondent would and would never vote for, for most responses.", "source": "Data Records, User Responses Dataset; Table 1", "quote": "This raw dataset provides n = 134,699 individual Likert-scale responses to a series of 39 policy issues and value statements along with additional user demographics and preferences for most of the responses", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "The raw file has 134,699 rows and every column of Table 1 and the data dictionary (69 columns). 81.0% of rows give demographics, 67.6% at least one salient statement, 73.2% a party considered and 76.9% a party never considered. 133,473 rows remain after removing those with -1, 133,143 after the R code's cleaning.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "collection-window", "statement": "The responses were gathered during the 26 days leading up to and including election day, from the launch on 5 September 2023 to the election on 30 September 2023.", "source": "Background & Summary, paragraph 3; Methods, Volebn\u00fd Kompas implementation and usage, paragraph 5", "quote": "gathered during the 26 days leading up to and including the day of the 2023 snap Slovak Parliamentary Elections", "statistics": [], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "The first response is 2023-09-05 11:26:31 and responses cover all 26 days to 30 September, but 432 of 134,699 rows (0.32%) are dated 1 to 9 October 2023 as given, and 494 (0.37%) when TIME.STAMP is read as UTC and converted to Slovak time; both readings cover all 26 days. The criterion requires every timestamp inside the window under one reading, and it fails under both; 99.7% of rows are inside it, so the claim holds in part. The file is sorted newest first, so the sample entry of Table 1 (2023-10-09 08:57:38) is its first row and the latest timestamp. The R code has an optional filter ending at 2023-09-30 22:42:30, which drops 451 rows of data_cleaned_corrected, the rows it is applied to (461 of the raw rows). The code parses the timestamps and the cut-off with tz \"UTC\", so it compares the values as given, and its comment gives the end of the moratorium as 22:45 with no time zone; this suggests local time (reading a) but does not settle it. The outcome is the same under both readings.", "deviations": [], "assumed": false, "disagreed": true, "data": "dataset", "where": null, "proposed": null }, { "claim": "party-positions-structure", "statement": "The Party Positions Dataset lists the 39 statements in Slovak, Hungarian and English and holds 429 party positions, both party self-placements and expert-coded positions, with sources, snippets and links.", "source": "Data Records, Party Positions Dataset", "quote": "This dataset lists the n = 39 statements used in the survey across three languages \u2014 Slovak, Hungarian, and English and offers detailed information on various parties\u2019 positions concerning specific statements with n = 429 observations", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "429 coded rows on 39 statements, each with one Slovak, one Hungarian and one English wording. Every row has expert positions of both coders and a final position; 131 rows have a self-placement. Snippet and link columns are present for self-placements, each coder and the final position. The \"Self-placement source\" column is empty in all rows.", "deviations": [ "Statements are numbered by their English wording, matched to Supplementary Table 1, since the workbook's \"Statement no.\" is wrong for 99 rows (see findings). Both numberings give 39 statements." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "eleven-parties", "statement": "Eleven parties are positioned on the 39 statements.", "source": "Methods, Party selection and coding, paragraph 1", "quote": "Out of 25 political parties, 11 met the inclusion criteria and were incorporated into our VAA", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "The workbook names 11 parties (ALIANCIA, DEMOKRATI, HLAS-SD, KDH, OLANO, PS, Republika, SME Rodina, SMER-SD, SNS, SaS), each with a position on all 39 statements; the recoded CSV holds the same 11.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "party-self-placements", "statement": "Four of the eleven parties returned the self-placement questionnaire.", "source": "Methods, Party selection and coding, paragraph 4", "quote": "Responses were garnered from four political entities out of eleven", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "Four parties have self-placements, SaS on 39 statements, DEMOKRATI on 38, PS on 32 and KDH on 22. PS used only \"agree\" and \"disagree\" (19 and 13), as a note in the workbook says.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "coder-agreement-kappa", "statement": "Agreement between the two expert coders of party positions was high, with a weighted Cohen's kappa of 0.84.", "source": "Methods, Party selection and coding, paragraph 3", "quote": "we employed Weighted Cohen\u2019s Kappa37, which provided a value of \u03ba = 0.84, indicating a high level of consistency among coders", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "The workbook holds each coder's positions. With quadratic weights and pairs where either coder chose \"No opinion\" left out (266 of 429), kappa is 0.842. Linear weights give 0.734 (No opinion left out) and 0.620 (No opinion as Neutral); quadratic weights with No opinion as Neutral give 0.745. Only the first reading is within 0.84 +- 0.084. The paper names neither the weights nor the handling of No opinion; this is asked of the authors (findings).", "deviations": [ "Assumed quadratic weights with \"No opinion\" pairs left out, the one reading of the four the criterion names that gives the reported value; the paper says only \"Weighted Cohen's Kappa\" (Methods, Party selection and coding, paragraph 3). Quadratic weights are the common choice for ordinal scales, and No opinion has no place on the ordered scale, so this reading is the best supported; all four are shown." ], "assumed": true, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "uni-and-landscape-placements", "statement": "The platform placed each party on five uni-dimensions, using 18 of the 39 statements, and on a two-dimensional socio-economic and socio-cultural landscape from 0 to 100, using 16 statements; the placements (Figs. 2 and 3) conform to the anticipated placements of Slovak parties.", "source": "Methods, Volebn\u00fd Kompas implementation and usage, paragraphs 3 and 4; Figs. 2 and 3; Technical Validation, Content validation, last paragraph", "quote": "Out of the 39 statements, 18 were assigned to the five uni-dimensions ... Of the 39 statements, 16 influenced this two-dimensional political landscape", "statistics": [], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "Supplementary Table 1, which is in the dataset, assigns 18 statements to the five uni-dimensions and 16 to the landscape (11 economic, 6 cultural, statement 10 on both). With No opinion as 50 (reading ii), every party's score on each of the five uni-dimensions is within 0.8 of Fig. 2, which places parties at whole percentages, and its cultural score is within 0.3 of Fig. 3. On the economic axis 10 of 11 parties are within 0.2 of Fig. 3, but PS scores 47.7 where Fig. 3 places it at 54.5, 38 pixels right of the centre line, so PS and DEMOKRATI (52.3, figure 52.2) are ordered differently. Leaving out statement 10 (on both axes) or 21 does not explain it; PS then scores 52.5 or 47.5, still below DEMOKRATI (55.0), and other parties move away from the figure. With No opinion left out (reading i) four of the five uni-dimensions differ from Fig. 2. The order matches on six of the seven axes, so the claim holds in part. The figure values were read blind to the computed scores, by pixel position (round 1 of verification asked for this). The paper's statement that the placements conform to anticipated placements is a judgement the data cannot test; the criterion tests agreement with the figures. The dimension assignments come from the dataset's own Supplementary Table 1, so no outside labels and no link claim are needed.", "deviations": [ "The expected placements were read from the published Figures 2 and 3 by a separate agent that had not seen the computed scores, from each logo's centre in pixels interpolated between the axis ends (Fig. 2 to about 0.3, Fig. 3 to about 0.3 to 1). A party matches within 1 point, and a pair is ordered differently only when the figure and the scores order it opposite ways by more than 1 point each.", "Readings (a) and (b) of the criterion give the same number for a mean of polarity-adjusted positions, so only one is computed.", "Statements are numbered by their wording, not by the workbook's \"Statement no.\", which is wrong for statements 16 to 24." ], "assumed": false, "disagreed": false, "data": "with_labels", "where": null, "proposed": null }, { "claim": "party-clustering", "statement": "Hierarchical clustering of the parties' recoded positions, by Manhattan distance and average linkage, groups the parties largely as anticipated (Fig. 7).", "source": "Technical Validation, Content validation, paragraph 7; Fig. 7", "quote": "We used the \u2018recoded_party_positions\u2019 dataset and computed pairwise Manhattan distances to emphasize the absolute differences in policy positions, applying average linkage clustering", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "With the recoded positions (No opinion as 50, reading a), Manhattan distances and average linkage, all 9 non-trivial clusters of Fig. 7 are found and no other, at heights within 1 of those read from the figure (root 1774.0, figure 1774). The figure was read blind to the computed tree, by pixel position against its axis ticks. Reading (b), No opinion left out of each distance, finds 7 of 9 clusters; the variants on coder I's and coder II's positions find 2 and 7 of 9. The deposited R code clusters the recoded CSV, as reading (a) does.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "table2-gender", "statement": "Of the responses counted in Table 2, 41.61% are female (44,919), 57.08% male (61,626) and 1.32% other or prefer not to say (1,420).", "source": "Table 2, Gender rows", "quote": "", "statistics": [], "kind": "finding", "inferred": "The numbers are only in Table 2; no sentence of the text states them.", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "Every reading is within tolerance. Reading (a), the paper's cleaning with gender given, gives 44,938, 61,670 and 1,455 of 108,063 (largest difference 2.5%). Gender, age and education are empty in the same 25,639 rows, so readings (c) and (d) equal (a) and (b). The R code's cleaning, which also drops 330 rows that give one answer to all 39 statements, gives Table 2 exactly (44,919, 61,626, 1,420 of 107,965).", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "table2-age", "statement": "Respondents are skewed toward younger ages; 32.32% are 25 to 34 (34,894) and 25.79% are 35 to 44 (27,847), with 2,716 aged 65 to 74 and 683 aged 75 or older (Table 2).", "source": "Technical Validation, Content validation, paragraph 2; Table 2, Age rows", "quote": "the substantial numbers within the younger age brackets, particularly those aged 25\u201334 (n = 34,894, 32.32%) and 35\u201344 (n = 27,847, 25.79%), highlight this trend (Table 2)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "Reading (a) gives every count within 2.4% of Table 2 and the two percentages within 0.01 point; every reading is within tolerance. The R code's cleaning gives every age count of Table 2 exactly.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "table2-education", "statement": "Respondents are skewed toward the well educated; 64.41% have a university degree or academic higher education (69,542), with 990 reporting no education and 1,665 only primary education (Table 2).", "source": "Technical Validation, Content validation, paragraph 2; Table 2, Education rows", "quote": "The pronounced presence of individuals with university degrees or academic higher education (n = 69,542, 64.41%)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "Reading (a) gives every count within 2.6% of Table 2 (no education 1,015 against 990); every reading is within tolerance. The R code's cleaning gives every education count exactly.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "age-education-inconsistencies", "statement": "The multivariate check found 40 respondents under 18 and 400 aged 18 to 20 reporting university-level education, which is not congruent with their age.", "source": "Technical Validation, Structural validation, paragraph 3", "quote": "specifically n = 40 cases under the age of 18 and n = 400 cases aged \u201818\u201320\u2019, which were not congruent with university-level education attainment", "statistics": [], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "With the university category only, 27 respondents under 18 and 378 aged 18-20 on the cleaned rows, and 27 and 382 on the raw rows; with applied higher education added, 34 and 505, and 34 and 509; the R code's cleaning gives 25 and 375, or 32 and 500. The 18-20 count is within tolerance under reading (i), but no reading brings the under-18 count within 4 of 40, so one of the two counts holds. The deposited R code's validation rules check only that AGE and EDUCATION take valid categories; no rule relates them. The paper says that upon initial review the team broadened the acceptable age range for a university degree to include 21-24, and that the 40 and 400 discrepancies \"were acknowledged during further data curation and refinement of the data collection tool\", so the counts may have come from an earlier state of the data; this is asked of the authors (findings) and of a reader (questions/figshare-24949470.yaml).", "deviations": [], "assumed": false, "disagreed": true, "data": "dataset", "where": null, "proposed": null }, { "claim": "s2-s3-wide-range", "statement": "Opinions on S2 (higher earners should face a higher income tax rate) and S3 (the retirement age should rise with life expectancy) span a wide range, with means of 43 and 42, standard deviations of 33 and medians of 25.", "source": "Technical Validation, Content validation, paragraph 4; Fig. 5", "quote": "S2 \u2014 \u2018People who earn higher wages should have a higher income tax rate\u2019 (mean = 43, s.d. = 33, median = 25) and S3\u2014 \u2018The retirement age should rise as life expectancy increases\u2019 (mean = 42, s.d. = 33, median = 25) highlight the diversity of opinions", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "On the cleaned rows, with No opinion left out, S2 has mean 43.77, s.d. 33.82, median 25 (130,235 answers) and S3 mean 42.09, s.d. 33.69, median 25. On the raw rows the S2 median is 50, outside the tolerance, and the other five values are within it.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "s4-s5-consensus", "statement": "S4 (more personal responsibility in health care) and S5 (heavier taxes on banks and big corporations) show stronger consensus, with medians of 75 (means 70 and 69, standard deviations 26 and 29), though more than 17,435 respondents for S4 and 23,344 for S5 disagree with scores of 0 or 25.", "source": "Technical Validation, Content validation, paragraph 4; Fig. 5", "quote": "S4 \u2014 \u2018In the health care system, more emphasis should be placed on the personal responsibility of the individual\u2019 (mean = 70, s.d. = 26, median = 75,) and S5 \u2014\u2018Banks and big corporations should be taxed more heavily\u2019 (mean = 69, s.d. = 29, median = 75) suggest a stronger consensus ... more than n = 17,435 for S4 and n = 23,344 for S5 \u2014 diverging on these views, with scores of 0 and 25", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "On the cleaned rows S4 has mean 70.36, s.d. 26.58, median 75 and 17,460 answers of 0 or 25, and S5 mean 69.38, s.d. 29.53, median 75 and 23,369, so the counts are within tolerance (reading i) and above the paper's (reading ii). The raw rows also hold. The R code's cleaning gives exactly 17,435 and 23,344, the paper's numbers, so \"more than\" in the paper reads as \"as many as\" for that selection.", "deviations": [], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "efa-eleven-factors", "statement": "Parallel analyses on both the polychoric and the Pearson correlation matrices of S1 to S39 indicated that 11 factors should be extracted.", "source": "Technical Validation, Content validation, paragraphs 5 and 6; Fig. 6", "quote": "Both parallel analyses indicated that 11 factors should be extracted.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "On the cleaned rows with No opinion as 50 (reading a), parallel analysis with psych's defaults (minres, 20 iterations) gives 11 factors for both the polychoric and the Pearson matrix. The 11th Pearson factor value is 0.030 against a reference of 0.013 and the 12th 0.004 against 0.012. The 12th polychoric value, 0.0106, is close to the normal-data reference (0.0117), but psych's own reference for cor poly, polychoric correlations of resampled data, gives 0.0143 (s.d. 0.0009 over 20 iterations) and again 11. The 95th percentile of each reference gives the same counts. Reading (b), No opinion missing, gives 12 (Pearson) and 11 (polychoric, against either reference) with the defaults, so with the defaults the claim holds under reading (a) only. With the R code's settings (ML extraction, 1000 iterations for Pearson and 100 for polychoric, on a correlation matrix, for which psych simulates normal data) every row reading gives 11 and 11, including the code's unique-email rows (29,493). 18 of 2,040 uniqueness fits on simulated reference matrices did not report convergence; every fit on an observed matrix (12) and on resampled data (40) converged.", "deviations": [ "The parallel analysis is a reimplementation of psych's fa.parallel in Python (factor values of the correlation matrix with one-factor communalities on the diagonal, compared with the mean over simulated normal data, and for the polychoric matrix under the defaults also over resampled data with polychoric correlations, as psych does for cor poly on raw data), not psych itself, which cannot be installed in the entry's Python environment.", "The R code's settings (fm \"ml\", n.iter 1000 and 100, unique-email rows) are shown as variants, taken from Volebny_Kompas_code.R (section \"CORRELATION AND FACTOR ANALYSIS\"); the criterion names psych's defaults.", "Polychoric correlations use thresholds from each statement's margins without psych's correction for empty cells." ], "assumed": false, "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "efa-loadings-consistent", "statement": "With 11 factors and promax rotation, maximum likelihood and principal axis factoring gave consistent results, and S1 (real estate tax raised to lower labour taxes) and S3 (retirement age) loaded strongly.", "source": "Technical Validation, Content validation, paragraph 6", "quote": "various estimation strategies were employed, including Maximum Likelihood (ML) and principal axis factoring, with promax rotation. This approach yielded consistent results. Notably statements such as S1 \u2014 \u2018Real estate tax should be increased to lower labor taxes rises\u2019 and S3 \u2014 \u2018The retirement age should rise as life expectancy rises\u2019 had strong loadings", "statistics": [], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_mintal_2024.ipynb", "notes": "On the Pearson matrix of the cleaned rows (reading a), S1 has a largest loading of 0.523 (ML) and 0.502 (PA) and S3 0.598 and 0.603, and 35 of 39 statements load most on matched factors (lowest Tucker congruence 0.892), so the claim holds there at both thresholds; reading (b) gives S1 0.553 and 0.539, S3 0.598 and 0.603, 35 of 39. On the polychoric matrices S1 and S3 load above 0.5 but only 32 of 39 statements match (lowest congruence 0.355 and 0.358), so the solutions are not consistent there. The criterion takes the matrices of efa-eleven-factors, Pearson and polychoric, so the claim holds on one of the two, in part. The R code runs ML and PA with promax only on the Pearson matrix; on its unique-email rows the Pearson PA loading of S1 is 0.379 and 33 of 39 match.", "deviations": [ "The thresholds for \"strong\" (0.4 or 0.5) and \"consistent\" (35 of 39) are the claims file's own; the paper gives none, so the outcome rests on them.", "ML, principal axis and promax are reimplemented in Python after psych and stats::promax." ], "assumed": true, "disagreed": true, "data": "dataset", "where": null, "proposed": null }, { "claim": "selected-parties-top-vote", "statement": "The 11 selected parties were the top 11 vote recipients in the 2023 election; each received at least 2% of the vote and no excluded party reached 1%.", "source": "Methods, Party selection and coding, paragraph 1", "quote": "The inclusion closely corresponded to election results, correctly anticipating the top 11 vote recipients. All of the 11 selected parties received at least 2% of the vote while no excluded party reached the 1%.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "other", "notebook": null, "notes": "The official results of the 30 September 2023 election are public on the Statistical Office's election site (volby.statistics.sk), which has no DOI or versioned release, so they cannot be pinned as the protocol requires. The ParlGov 2024 release (doi:10.7910/DVN/2VZ5ZC), the pinned source looked at, ends its Slovak elections with 2020 in both its stable and experimental databases; searches of Zenodo and Harvard Dataverse found no pinned copy. These lookups were made while preparing and are not in the notebook.", "deviations": [], "assumed": false, "disagreed": false, "data": "elsewhere", "where": "Official results of the 30 September 2023 Slovak parliamentary election (Statistical Office of the Slovak Republic); the paper cites no source for them beyond ref. 20.", "proposed": null }, { "claim": "website-views", "statement": "The platform registered 270,836 website views in the 26 days leading up to the election.", "source": "Methods, Volebn\u00fd Kompas implementation and usage, paragraph 6; Fig. 4", "quote": "registering 270,836 website views within the 26-day span leading up to the elections", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "other", "notebook": null, "notes": "The view counts are the platform's usage records (euandi2019, run by the European University Institute). The paper does not deposit them, the Figshare item does not hold them, and no public copy is named anywhere; the responses cannot stand in for views. The claims file marks the data as elsewhere, so `data_missing` is not used.", "deviations": [], "assumed": false, "disagreed": false, "data": "elsewhere", "where": "The platform's usage records (the euandi2019 platform run by the Robert Schuman Centre, European University Institute); the paper does not say they are deposited.", "proposed": null } ], "findings": [ { "kind": "metadata_error", "statement": "In \"VAA_party_positions coded.xlsx\" the \"Statement no.\" column disagrees with the statement's wording in 99 of 429 rows. For every party, the rows worded as statements 16 to 24 (Supplementary Table 1 and the data dictionary) are numbered 22, 23, 24, 16, 17, 18, 19, 20, 21, so the environment, multilevel governance and form of government statements are mislabelled. The recoded CSV and the paper's Figures 2 and 3 follow the wording. A user who joins positions to the user responses by \"Statement no.\" gets the wrong statements.", "found": null, "status": null }, { "kind": "selection_undocumented", "statement": "The deposited R code removes 330 responses that give the same answer to all 39 statements (204 of them all \"no opinion\") before removing incomplete ones; the paper's Data cleaning does not mention this step. With it, Table 2 and the S4 and S5 counts are reproduced exactly; without it they differ by up to 2.5%.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "The paper says the responses were gathered during the 26 days up to and including election day, but 432 rows (0.32%) are timestamped 1 to 9 October 2023, after the election. Table 1's sample entry (2023-10-09 08:57:38) is one of them. The R code has an optional filter that drops rows after 2023-09-30 22:42:30, but the deposited file includes them and the paper does not say so; the filter drops 451 of the rows it is applied to.", "found": null, "status": null }, { "kind": "metadata_error", "statement": "The paper's Data cleaning says \"no opinion\" answers are empty strings; in data_without_email.csv they are the string \"NA\" (183,459 values in S1 to S39). The README calls each row \"a unique respondent\", while EMAIL_ID repeats (30,775 distinct ids over 32,009 rows with an id), as the paper says it may.", "found": null, "status": null }, { "kind": "metadata_insufficient", "statement": "The data dictionary lists no categories for GENDER, AGE and EDUCATION (they appear only in the R code's validation rules), does not give the time zone of TIME.STAMP, and lists a \"Self-placement source\" column that is empty in all 429 rows.", "found": null, "status": null }, { "kind": "other", "statement": "Figure 3 places PS at 54.5 on the economic axis, right of centre, while the deposited final positions give 47.7 by the same calculation that reproduces every other party's placement in Figures 2 and 3 to within 0.8. The figure and the deposited positions disagree for this one party.", "found": null, "status": null }, { "kind": "methods_unavailable", "statement": "Question for the paper's authors. The weighted Cohen's kappa of 0.84 (Methods, Party selection and coding, paragraph 3) does not state the weights or how \"No opinion\" codes enter. Quadratic weights with pairs involving \"No opinion\" left out give 0.842; linear weights give 0.734, and scoring \"No opinion\" as neutral gives 0.620 (linear) or 0.745 (quadratic). This attempt assumed quadratic weights with \"No opinion\" left out; please confirm or correct.", "found": null, "status": null }, { "kind": "methods_unavailable", "statement": "Question for the paper's authors. The structural validation reports 40 respondents under 18 and 400 aged 18-20 with university-level education (Technical Validation, Structural validation, paragraph 3). The deposited data give 27 and 378 (cleaned rows, university only), 27 and 382 (raw), or 34 and 505 with applied higher education. Which rows and categories were counted, and were the counts made on an earlier state of the data? The R code does not contain this check.", "found": null, "status": null } ], "reconciliation": null, "verifications": [ { "date": "2026-10-27", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The executed notebook of commit bcc64466 (author-answers) with the conclusion cells and the summary removed, the helper module and the claims file, with the paper. The verifier also listed the item's files through the Figshare API. agrees compares its outcome with the outcome of the record it reviewed.", "agreed": 15, "of": 18, "disagreements": [ { "claim": "collection-window", "outcome": "partially_reproduced", "reason": "Cell 11, responses start 2023-09-05 and cover all 26 days of the window. But 432 rows under reading (a) and 494 under (b) fall between 1 and 9 October, so the criterion (every timestamp inside the window) fails under both readings. Problem 9, the R code's moratorium cutoff (2023-09-30 22:42:30, as given) suggests TIME.STAMP is local time, which makes reading (b) unlikely; under either reading, rows still fall after election day." }, { "claim": "age-education-inconsistencies", "outcome": "partially_reproduced", "reason": "Cell 29, the 18-20 count is 378 (cleaned) or 382 (raw) with university only, inside 400 +- 40. The under-18 count is 25 to 34 under every reading, never inside 40 +- 4. Problem 2, the R code's validate rule for age and education, which this claim depends on, is never examined. Problem 8, the paper says these inconsistencies were found upon initial review and then acknowledged during further data curation, so the 40 and 400 may describe an earlier state of the data, and the criterion does not mention this possibility." }, { "claim": "efa-loadings-consistent", "outcome": "partially_reproduced", "reason": "Cell 37, on Pearson, readings (a) and (b), S1 and S3 load at 0.50 or more in both solutions and 35 of 39 statements match, so the claim holds under the 0.4 and 0.5 thresholds. On polychoric only 32 of 39 match (lowest congruence 0.36), so consistency fails. The criterion says matrices, plural, and the thresholds are the claims file's own." } ] }, { "date": "2026-10-27", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The full executed notebook of commit 75391ac8 (author-revise-1), with its conclusions and summary, the helper module, the claims file, the attempt record, the README and the paper. agrees compares its outcome with the outcome of the record it reviewed.", "agreed": 18, "of": 18, "disagreements": [] } ] } ], "entries": [ { "name": "figshare-24949470-mintal-2024", "type": "reproduction", "title": "The Volebny Kompas datasets on Slovak voter and party positions (Mintal et al. 2024)", "description": "Tests the findings of Mintal et al. (2024) on Figshare item 24949470, version 1, which holds 134,699 responses to the Volebny Kompas voting advice application before the 2023 Slovak election, the positions of 11 parties on its 39 statements, the paper's R code and Supplementary Table 1. It checks the counts and structure of both datasets, the coders' agreement, the platform's placement of the parties and their clustering, the demographic table, the summary statistics and the factor analysis.", "datasets": [ "figshare:24949470" ], "papers": [ "10.1038/s41597-024-03777-0" ], "derived_from": [], "notebooks": [ { "path": "reproduce_mintal_2024.ipynb", "title": "Reproducing Mintal et al. (2024) from Figshare item 24949470", "executed": "notebooks/figshare-24949470-mintal-2024/reproduce_mintal_2024.html" } ] } ] }