@article{970b24ce69a347078d87cf991533c4e0,
title = "Improving the Reliability of Cognitive Task Measures: A Narrative Review",
abstract = "Cognitive tasks are capable of providing researchers with crucial insights into the relationship between cognitive processing and psychiatric phenomena. However, many recent studies have found that task measures exhibit poor reliability, which hampers their usefulness for individual differences research. Here, we provide a narrative review of approaches to improve the reliability of cognitive task measures. Specifically, we introduce a taxonomy of experiment design and analysis strategies for improving task reliability. Where appropriate, we highlight studies that are exemplary for improving the reliability of specific task measures. We hope that this article can serve as a helpful guide for experimenters who wish to design a new task, or improve an existing one, to achieve sufficient reliability for use in individual differences research.",
keywords = "Behavioral tasks, Cognitive functions, Computational psychiatry, Individual differences, Psychometrics, Reliability",
author = "Samuel Zorowitz and Yael Niv",
note = "Funding Information: This project was made possible with support from the National Center for Advancing Translational Sciences , a component of the National Institutes of Health , under Grant No. UL1TR003017a (to YN) and a National Science Foundation Graduate Research Fellowship (to SZ). Funding Information: This project was made possible with support from the National Center for Advancing Translational Sciences, a component of the National Institutes of Health, under Grant No. UL1TR003017a (to YN) and a National Science Foundation Graduate Research Fellowship (to SZ). We thank Rachel Bedder and Felix Jan Nitsch for helpful feedback on this manuscript. The code used to generate the figures in this manuscript is publicly available at https://github.com/nivlab/biopsych-reliability-review. Citation Diversity Statement: Recent work in several fields of science has identified a bias in citation practices such that papers from women and other minority scholars are under-cited relative to the number of such papers in the field (81,82). Here, we sought to proactively consider choosing references that reflect the diversity of the field in thought, form of contribution, gender, race, ethnicity, and other factors. First, we obtained the predicted gender of the first and last author of each reference by using databases that store the probability of a first name being carried by a woman (81). By this measure, and excluding self-citations to the first and last authors of our current article, our references contain 8.3% woman (first)/woman (last), 14.7% man/woman, 18.6% woman/man, and 58.4% man/man. This method is limited in that 1) names, pronouns, and social media profiles used to construct the databases may not, in every case, be indicative of gender identity; and 2) it cannot account for intersex, nonbinary, or transgender people. Second, we obtained predicted racial/ethnic category of the first and last author of each reference by databases that store the probability of a first and last name being carried by an author of color (83,84). By this measure (and excluding self-citations), our references contain 4.7% author of color (first)/author of color (last), 15.5% White author/author of color, 18.0% author of color/White author, and 61.8% White author/White author. This method is limited in that 1) names and Florida Voter Data to make the predictions may not be indicative of racial/ethnic identity and 2) it cannot account for Indigenous and mixed-race authors, or those who may face differential biases due to the ambiguous racialization or ethnicization of their names. We look forward to future work that could help us to better understand how to support equitable practices in science. The authors report no biomedical financial interests or potential conflicts of interest. Publisher Copyright: {\textcopyright} 2023 Society of Biological Psychiatry",
year = "2023",
doi = "10.1016/j.bpsc.2023.02.004",
language = "English (US)",
journal = "Biological Psychiatry: Cognitive Neuroscience and Neuroimaging",
issn = "2451-9022",
publisher = "Elsevier Inc.",
}