-
Notifications
You must be signed in to change notification settings - Fork 9
/
_main.tex
2083 lines (1653 loc) · 138 KB
/
_main.tex
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
\PassOptionsToPackage{unicode=true}{hyperref} % options for packages loaded elsewhere
\PassOptionsToPackage{hyphens}{url}
%
\documentclass[]{book}
\usepackage{lmodern}
\usepackage{amssymb,amsmath}
\usepackage{ifxetex,ifluatex}
\usepackage{fixltx2e} % provides \textsubscript
\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{textcomp} % provides euro and other symbols
\else % if luatex or xelatex
\usepackage{unicode-math}
\defaultfontfeatures{Ligatures=TeX,Scale=MatchLowercase}
\fi
% use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
% use microtype if available
\IfFileExists{microtype.sty}{%
\usepackage[]{microtype}
\UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\IfFileExists{parskip.sty}{%
\usepackage{parskip}
}{% else
\setlength{\parindent}{0pt}
\setlength{\parskip}{6pt plus 2pt minus 1pt}
}
\usepackage{hyperref}
\hypersetup{
pdftitle={BITSS logo Guidelines for Verification of Computational Reproducibility in Economics},
pdfauthor={ACRE Team},
pdfborder={0 0 0},
breaklinks=true}
\urlstyle{same} % don't use monospace font for urls
\usepackage{color}
\usepackage{fancyvrb}
\newcommand{\VerbBar}{|}
\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
% Add ',fontsize=\small' for more characters per line
\usepackage{framed}
\definecolor{shadecolor}{RGB}{248,248,248}
\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\BuiltInTok}[1]{#1}
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
\newcommand{\ExtensionTok}[1]{#1}
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\ImportTok}[1]{#1}
\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
\newcommand{\NormalTok}[1]{#1}
\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
\newcommand{\RegionMarkerTok}[1]{#1}
\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\usepackage{longtable,booktabs}
% Fix footnotes in tables (requires footnote package)
\IfFileExists{footnote.sty}{\usepackage{footnote}\makesavenoteenv{longtable}}{}
\usepackage{graphicx,grffile}
\makeatletter
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
\makeatother
% Scale images if necessary, so that they will not overflow the page
% margins by default, and it is still possible to overwrite the defaults
% using explicit options in \includegraphics[width, height, ...]{}
\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
\setlength{\emergencystretch}{3em} % prevent overfull lines
\providecommand{\tightlist}{%
\setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
\setcounter{secnumdepth}{5}
% Redefines (sub)paragraphs to behave more like sections
\ifx\paragraph\undefined\else
\let\oldparagraph\paragraph
\renewcommand{\paragraph}[1]{\oldparagraph{#1}\mbox{}}
\fi
\ifx\subparagraph\undefined\else
\let\oldsubparagraph\subparagraph
\renewcommand{\subparagraph}[1]{\oldsubparagraph{#1}\mbox{}}
\fi
% set default figure placement to htbp
\makeatletter
\def\fps@figure{htbp}
\makeatother
\usepackage{booktabs}
\usepackage{booktabs}
\usepackage{longtable}
\usepackage{array}
\usepackage{multirow}
\usepackage{wrapfig}
\usepackage{float}
\usepackage{colortbl}
\usepackage{pdflscape}
\usepackage{tabu}
\usepackage{threeparttable}
\usepackage{threeparttablex}
\usepackage[normalem]{ulem}
\usepackage{makecell}
\usepackage{xcolor}
\usepackage[]{natbib}
\bibliographystyle{apalike}
\title{\href{https://www.bitss.org/}{\includegraphics{BITSS_logo_horizontal.png}}\\
Guidelines for Verification of Computational Reproducibility in Economics}
\author{ACRE Team}
\date{2020-06-26}
\begin{document}
\maketitle
{
\setcounter{tocdepth}{1}
\tableofcontents
}
\hypertarget{section}{%
\chapter*{}\label{section}}
\addcontentsline{toc}{chapter}{}
\protect\hyperlink{contributions}{See a full list of contributors}
\begin{verbatim}
(1) (2) (3) (4)
scope --> assess --> improve --> robust
▲ | | ▲
| | | |
|_________| |___________________|
\end{verbatim}
\protect\hyperlink{scoping}{(1) Scoping}
\protect\hyperlink{assessment}{(2) Assessment}
\protect\hyperlink{improvements}{(3) Improvement}
\protect\hyperlink{robust}{(4) Robustness}
\href{}{(5) Extensions}
\protect\hyperlink{improvements}{Outcome-level}
\protect\hyperlink{paper-level}{Paper-level}
☐ Select paper
\protect\hyperlink{describe-inputs}{☐ Describe inputs}
\protect\hyperlink{rd}{☐ + Raw data}
\protect\hyperlink{paper-level}{☐ + Version control}
\protect\hyperlink{id-analy}{☐ Analytical choices}
☐ New method
\protect\hyperlink{check-acre}{☐ Check ACRE}
\protect\hyperlink{diagram}{☐ Reproduction diagrams}
\protect\hyperlink{ad}{☐ + Analysis data}
\protect\hyperlink{paper-level}{☐ + Documentation}
\protect\hyperlink{id-type}{☐ Type of choice}
☐ New data
☐ \protect\hyperlink{verify-rep-mat}{Check Rep.~pkg exists}
\protect\hyperlink{score}{☐ Reproduction score}
\protect\hyperlink{ac}{☐ + Analysis code}
\protect\hyperlink{paper-level}{☐ + Dynamic document}
\protect\hyperlink{id-val}{☐ Choice value}
\protect\hyperlink{read-summ}{☐ Read paper}
\protect\hyperlink{cc}{☐ + Cleaning code}
\protect\hyperlink{paper-level}{☐ + File structure}
\protect\hyperlink{test-rob}{☐ Justify and test alternatives}
\protect\hyperlink{declare-estimates}{☐ Declare estimates}
\protect\hyperlink{dac}{☐ Debug analysis code}
\protect\hyperlink{dcc}{☐ Debug cleaning code}
\href{https://berkeley.qualtrics.com/jfe/form/SV_2bO83uJvU9ZiTXv}{Record results in Survey 1}
\href{https://berkeley.qualtrics.com/jfe/form/SV_2gd9Y3XVtjLpZL7}{Record results in Survey 2}
\href{ADD\%20LINK}{Record results in Survey 3}
\hypertarget{intro}{%
\chapter*{Introduction}\label{intro}}
\addcontentsline{toc}{chapter}{Introduction}
In 2019, the \href{https://www.aeaweb.org/journals/policies/data-code/}{American Economic Association} updated its Data and Code Availability Policy to require that the AEA Data Editor verify the reproducibility of all papers before they are accepted by an AEA journal. In addition to the requirements laid out in the policy, several \href{https://aeadataeditor.github.io/aea-de-guidance/}{specific recommendations} were produced to facilitate compliance. This change in policy is expected to improve the computational reproducibility of all published research going forward, after several studies showed that rates of \emph{computational reproducibility} in economics at large range from somewhat low to alarmingly low \citep{galiani2018make, chang2015economics, kingi2018reproducibility}.
\emph{Replication}, or the process by which a study's hypotheses and findings are re-examined using different data or different methods (or both) \citep{King95} is an essential part of the scientific process that allows science to be ``self-correcting.'' \emph{Computational reproducibility}, or the ability to reproduce the results, tables, and other figures of a paper using the available data, code, and materials, is a precondition for replication. Computational reproducibility is assessed through the process of \emph{reproduction}. At the center of this process is the \emph{reproducer} (you!), a party rarely involved in the production of the original paper. Reproductions sometimes involve the \emph{original author} (whom we refer to as ``the author'') in cases where additional guidance and materials are needed to execute the process.
This exercise is designed for reproductions performed in economics graduate courses or undergraduate theses, with the goal of providing a common approach, terminology, and standards for conducting reproductions. The goal of reproduction, in general, is to assess and improve the computational reproducibility of published research in a way that facilitates further robustness checks, extensions, collaborations, and replication.
This exercise is part of the Accelerating Computational Reproducibility in Economics \href{https://www.bitss.org/ecosystem/acre/}{(ACRE)} project, which aims to assess, enable, and improve the computational reproducibility of published economics research. The ACRE project is led by the Berkeley Initiative for Transparency in the Social Sciences \href{https://bitss.org}{(BITSS)}---an initiative of the Center for Effective Global Action \href{https://cega.berkeley.edu/}{(CEGA)}---and Dr.~Lars Vilhuber, Data Editor for the journals of the American Economic Association (AEA). This project is supported by the Laura and John Arnold Foundation.
\hypertarget{beyond-binary-judgments}{%
\section*{Beyond binary judgments}\label{beyond-binary-judgments}}
\addcontentsline{toc}{section}{Beyond binary judgments}
Assessments of reproducibility can easily gravitate towards binary judgements that declare an entire paper ``reproducible'' or ``non-reproducible.'' These guidelines suggest a more nuanced approach by highlighting two realities that make binary judgments less relevant.
First, a paper may contain several scientific claims (or major hypotheses) that may vary in computational reproducibility. Each claim is tested using different methodologies, presenting results in one or more display items (outputs like tables and figures). Each display item will itself contain several specifications. Figure \ref{fig:diagram} illustrates this idea.
\begin{figure}
\centering
\includegraphics{01-intro_files/figure-latex/diagram-1.pdf}
\caption{\label{fig:diagram}One paper has multiple components to reproduce. DI: Display Item, S: Specification}
\end{figure}
Second, for any given specification there are several levels of reproducibility, ranging from the absence of any materials to complete reproducibility starting from raw data. And even for a specific claim-specification, distinguishing the appropriate level can be far more constructive than simply labeling it as (ir)reproducible.
Note that the highest level of reproducibility, which requires complete reproducibility starting from raw data, is very demanding to achieve and should not be expected of all published research --- especially before 2019. Instead, this level can serve as an aspiration for the field of economics at large as it seeks to improve the reproducibility of research and facilitate the transmission of knowledge throughout the scientific community.
\hypertarget{stages-of-the-exercise}{%
\section*{Stages of the exercise}\label{stages-of-the-exercise}}
\addcontentsline{toc}{section}{Stages of the exercise}
This reproduction exercise is divided into four stages, corresponding to the first four chapters of these guidelines, with a fifth optional stage:
\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\item
\protect\hyperlink{scoping}{\textbf{Scoping}}, where you (the reproducer) will define the scope of the exercise by declaring a paper and the specific output(s) on which you will focus for the remainder of the exercise;\\
\item
\protect\hyperlink{assessment}{\textbf{Assessment}}, where you will review and describe in detail the available reproduction package, and assess the current level of computational reproducibility of the selected outputs;
\item
\protect\hyperlink{improvements}{\textbf{Improvement}}, where you will modify the content and/or the organization of the reproduction package to improve its reproducibility;\\
\item
\protect\hyperlink{robust}{\textbf{Robustness checks}}, where you will assess the quality of selected analytical choices; and\\
\item
\textbf{Extension} (if applicable), where you may extend the current paper by including new methodologies or data. This step brings the reproduction exercise a step closer to \emph{replication}.
\begin{verbatim}
Figure 2: Steps for reproduction
(1) (2) (3) (4) (5)
scope --> assess --> improve --> robust --> extend
▲ | | ▲
| | | |
|_________| |___________________|
Suggested level of effort:
- Graduate
research: 5% 10% 5% 10% 70%
- Graduate
course: 10% 25% 20% 40% 5%
- Undergrad
thesis: 10% 30% 40% 20% 0%
\end{verbatim}
\end{enumerate}
Figure 2 depicts suggested levels of effort for each stage of the exercise depending on the context in which you are performing a reproduction. This process need not be chronologically linear. For example, you may realize that the scope of a reproduction is too ambitious and switch to a less intensive one. Later in the exercise, you can also begin testing different specifications for robustness while also assessing a paper's level of reproducibility.
\hypertarget{recording-the-results-of-the-exercise}{%
\section*{Recording the results of the exercise}\label{recording-the-results-of-the-exercise}}
\addcontentsline{toc}{section}{Recording the results of the exercise}
You will be asked to record the results of your reproduction as you progress through each stage.
In \emph{Stage 1: \protect\hyperlink{scoping}{Scoping}}, complete \textbf{\href{https://berkeley.qualtrics.com/jfe/form/SV_2bO83uJvU9ZiTXv}{Survey 1}}, where you will declare your paper of choice and the specific display item(s) and specifications on which you will focus for the remainder of the exercise. This step may also involve writing a brief 1-2 page summary of the paper (depending on your instructor or goals).
In \emph{Stage 2: \protect\hyperlink{assessment}{Assessment}}, you will inspect the paper's reproduction package (raw data, analysis data, and code), connect the display item to be reproduced with its inputs, and assign a reproducibility score to each output.
In \emph{Stage 3: \protect\hyperlink{improvements}{Improvement}}, you will try to improve the reproducibility of the selected outputs by adding missing files, documentation, and report any potential changes in the level of reproducibility. Use \textbf{Survey 2} to record your work at Stages 2 and 3 (you will receive access instructions for Survey 2 when you submit Survey 1).
In \emph{Stage 4: \protect\hyperlink{robust}{Robustness Checks}}, you will assess different analytical choices and test possible variations. Use \textbf{Survey 3} to record your work at this stage.
\hypertarget{reproduction-strategies}{%
\section*{Reproduction Strategies}\label{reproduction-strategies}}
\addcontentsline{toc}{section}{Reproduction Strategies}
Generally, a reproduction will begin with a thorough reading of the study being reproduced. However, subsequent steps may follow from a \emph{reproduction strategy}. For example, a reproduction may closely follow the order of the steps outlined above. This might entail the reproducer first choosing a set of results whose pproduction they are interested in assessing or understanding, completely reproducing these results to the extent possible, and then making modifications to the reproduction package. Another potential strategy could be for the reproducer to develop potential robustness checks or extensions while reading the study, which would lead to the definition of a set of results to be assessed via reproduction. Yet another reproduction strategy may be for the reproducer to seek out a paper that uses a particular dataset to which they have access or an interest in using, reproducing the results that use that dataset as an input, then probing the robustness of the results to various data cleaning decisions.
The various uses of reproduction makes the number of potential reproduction strategies quite large. In choosing or designing a reproduction strategy, it is helpful to clearly identify the goal of the reproduction. In all of the examples laid out in the paragraph above, the order in which the steps of the reproduction exercise are taken is at least partially determined by what the reproducer hopes to get from the exercise. The structure provided in these guidelines, together with a clear reproduction goal, can facilitate the implementation of an efficient reproduction strategy.
\hypertarget{scoping}{%
\chapter{Scoping}\label{scoping}}
In this stage, you will define the scope of your exercise by declaring a paper and the specific output(s) on which you will focus. You might first consider multiple papers without analyzing them more closely (we refer to these as \textbf{candidate papers}) before moving forward with your \textbf{declared paper}.
It is likely that you will choose a declared paper based on whether or not you can locate its reproduction package. A \textbf{reproduction package} is the collection of materials that make it possible to reproduce a paper. This package may contain data, code, or documentation. If you are unable to independently locate the reproduction package for your paper, you can ask the paper's author for it (find guidance on this in \href{https://bitss.github.io/ACRE/guidance-for-a-constructive-exchange-between-reproducers-and-original-authors.html}{Chapter 6}) or simply choose another candidate paper. If you still want to explore the reproducibility of a paper with no reproduction package, these guidelines provide instructions for requesting materials from authors to create a public reproduction package, or if this proves unsuccessful, for building your reproduction package from scratch.
To avoid duplicating the efforts of others who may be interested in reproducing one of your candidate papers, \textbf{we ask that you record your candidate papers in the ACRE database} (currently under development).
Note that in this stage, \emph{you are not expected to review the reproduction materials in detail}, as you will dedicate most of your time to this in later stages of the exercise. If materials are available, you will read the paper and declare the scope of the reproduction exercise. You can expect to spend between 1-3 days in this Scoping stage, though this may vary based on the length and the complexity of the paper, and the availability of reproduction materials.
\textbf{\emph{Use \href{https://berkeley.qualtrics.com/jfe/form/SV_2bO83uJvU9ZiTXv}{Survey 1} to record your work in this stage.}}
\hypertarget{from-candidate-to-declared-paper}{%
\section{From candidate to declared paper}\label{from-candidate-to-declared-paper}}
At this point of the exercise, you are \emph{only validating the availability} of (at least) one reproduction package and not assessing the quality of its content. Follow the steps below to verify that a reproduction package is available, and stop whenever you find it (this may mean mean that you have found your declared paper).
\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
Check whether previous reproduction attempts have beeen recorded in the ACRE Database for the paper (more on the ACRE Database in the next section).
\item
Check the journal or publisher's website, looking for materials named ``Data and Materials,'' ``Supplemental Materials,'' ``Reproduction/Replication Package/Materials,'' etc.\\
\item
Look for links in the paper (review the footnotes and appendices).\\
\item
Review the personal websites of the paper's author(s).
\item
Contact the author(s) to request the reproduction package using \href{https://bitss.github.io/ACRE/guidance-for-a-constructive-exchange-between-reproducers-and-original-authors.html\#contacting-the-original-authors-when-there-is-no-reproduction-package}{this} email template. In this and future interactions with authors, we encourage you to follow our guidance outlined in \href{https://bitss.github.io/ACRE/guidance-for-a-constructive-exchange-between-reproducers-and-original-authors.html\#contacting-the-original-authors-when-there-is-no-reproduction-package}{Chapter 5}.
\item
Deposit the reproduction package in a trusted repository (e.g., \href{https://dataverse.org/}{Dataverse}, \href{https://www.openicpsr.org/openicpsr/}{Open ICPSR}, \href{https://zenodo.org/}{Zenodo}, or the \href{https://osf.io/}{Open Science Framework}) under the name \texttt{Original\ reproduction\ package\ for\ -\ Title\ of\ the\ paper}. You will be asked to provide the URL of the repository in Survey 1.
\end{enumerate}
In case you need to contact the authors, make sure to \emph{allocate sufficient time for this step} (we suggest at least three weeks before the date you plan to start the reproduction). Instructors should also plan to accordingly (e.g., if the ACRE exericse is expected to take place in the middle of the semester, students should review candidate papers and (if applicable) contact the authors in the first few weeks of the semester).
Review the decision tree (Figure \#) below for a more detailed overview of this process. Remember, \emph{if at any step of the process you decide to abandon the paper, make sure to record the candidate paper in the ACRE database} before moving on to another candidate paper. Once you have obtained the reproduction package, the \emph{candidate paper} becomes your \emph{declared paper} and you can move forward with the exercise! Do not invest time in doing a detailed read of any paper until you are sure that it is your declared paper.
\hypertarget{candidate-paper-entries-in-the-acre-database}{%
\subsection{Candidate paper entries in the ACRE Database}\label{candidate-paper-entries-in-the-acre-database}}
If the ACRE database contains previous reproduction attempts of the paper, you will see a report card with the following information:
\begin{quote}
\textbf{Box 1:} Summary Report Card for ACRE Paper Entry\\
\textbf{Title:} Sample Title\\
\textbf{Authors:} Jane Doe \& John Doe\\
\textbf{Original Reproduction Package Available:} URL/No {[}What does this mean? Add some context{]}.
{[}If ``No''{]} \textbf{Contacted Authors?:} Yes/No\\
{[}If ``Yes(contacted)''{]} \textbf{Type of Response:} Categories (6).\\
\textbf{Additional Reproduction Packages:} Number (eg., 2)\\
\textbf{Authors Available for Further Questions for ACRE Reproductions:} Yes/No/Unknown\\
\textbf{Open for reproductions:} Yes/No {[}Same as above: what does this mean? Add more context{]}.
\end{quote}
If after taking steps 1-5 above (or for some other reason) you are unable to locate the reproduction package, record your candidate paper (and if applicable, the outcome of your correspondence with the original authors) in the ACRE database following the example above.
View Decision Tree To Select Paper (Emma: add title and solve bug with svg)
\includegraphics{candidate-to-declared-paper.png}
\hypertarget{scoping-your-declared-paper}{%
\section{Scoping your declared paper}\label{scoping-your-declared-paper}}
Once you have identified your declared paper, get familiarized with it and choose the specific output(s) on which you will focus for the remainder of the exercise.
\hypertarget{read-sum}{%
\subsection{Read and summarize the paper}\label{read-sum}}
Depending on how much time you have, we recommend that you write a short (1-2 page) summary of the paper. This will help remind you of the key elements to focus on for the reproduction, and demonstrate your understanding of the paper (for yourself and others like your instructor or advisor).
When reading or summarizing the paper, try to answer the following questions:
\begin{itemize}
\tightlist
\item
Would you classify the paper's scientific claims as mainly focused on estimating a causal relationship, estimating/predicting a descriptive statistic of a population, or something else?
\item
How many scientific claims (descriptive or causal) are investigated in the paper?
\item
What is the population for which the estimates apply?
\item
What is the population that is the focus of the paper as a whole?
\item
What are the main data sources used in the paper?
\item
How many display items are there in the paper (tables, figures, and inline results)?\\
\item
What is the main statistical or econometric method used to examine each claim?
\item
What is the author's preferred specification (or yours, if the authors are not clear)?
\item
What are some robustness checks for the preferred specification?
\end{itemize}
\hypertarget{declare-estimates}{%
\subsection{Record scope of the exercise}\label{declare-estimates}}
By now you should have a fairly good understanding of the paper's content. You do not, however, need to have spent any time reviewing the reproduction package in detail.
At this point, you should clearly specify which part of the paper will be the main focus of your reproduction. Focus on specific estimates, represented by a unique combination of claim-display item-specification as represented in figure \ref{fig:diagram}. If you plan to scope more than one claim, \emph{we strongly recommend starting with just one} and recording your results. You can then initiate another record in ACRE later for the second (or third, fourth, etc.) claim to reproduce using the materials and knowledge you developed in the first exercise. You can, however, reproduce more than one claim if you are already familiar with the paper.
In the Assessment stage, the reproduction will be centered around the display item(s) that contain the specification you indicate at this point.
\hypertarget{declare-specific-main-estimates-to-reproduce.}{%
\subsubsection*{Declare specific main estimates to reproduce.}\label{declare-specific-main-estimates-to-reproduce.}}
\addcontentsline{toc}{subsubsection}{Declare specific main estimates to reproduce.}
Identify a scientific claim and its corresponding preferred specification, and record its magnitude, standard error, and location in the paper (page, table \#, and table row and column). If the authors did not explicitly chose a particular estimate, you will be asked to select one. In addition to the preferred estimate, reproduce up to five estimates that correspond to alternative specifications of the preferred estimate.
\hypertarget{declare-possible-robustness-checks-for-main-estimates-optional.}{%
\subsubsection*{Declare possible robustness checks for main estimates (optional).}\label{declare-possible-robustness-checks-for-main-estimates-optional.}}
\addcontentsline{toc}{subsubsection}{Declare possible robustness checks for main estimates (optional).}
After reading the paper, you might wonder why the authors did not conduct a specific robustness test. If you think that such analysis could have been done \emph{within the same methodology} and \emph{using the same data} (e.g., by including or excluding a subset of the data like ``high-school dropouts'' or ``women''), please specify a robustness test that you would like to conduct before starting the Assessment stage.
These are the elements you will need for the Scoping stage. \textbf{You now have all the elements necessary to complete \href{https://berkeley.qualtrics.com/jfe/form/SV_2bO83uJvU9ZiTXv}{Survey 1}.}
\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center}
\hypertarget{identify-your-relevant-timeline.}{%
\section{Identify your relevant timeline.}\label{identify-your-relevant-timeline.}}
Before you begin working on the three main stages of the reproduction exercise (Assessment, Improvement, and Robustness), it is important to manage your own expectations and those of your instructor or advisor. Be mindful of your time limitations when defining the scope of your reproduction activity. These will depend on the type of exercise chosen by your instructor or advisor and may vary from a weeklong homework assignment, to a longer class project that may take a month to complete or a semester-long project (an undergraduate thesis, for example).
Table 1 shows an example distribution of time across three different reproduction formats. The Scoping and Assessment stages are expected to last roughly the same amount of time across all formats (lasting longer for the semester-long activities, and acknowledgin that less experienceed researchers, such as undergraduate students, may need more time). Differences emerge in the distribution of time for the last two main stages: Improvements and Robustness. For shorter exercises, we recommend avoiding any possible improvements to the raw data (or cleaning code). This will limit how many robustness checks are possible (for example, by limiting your ability to reconstruct variables according to slightly different definitions), but it should leave plenty of time for testing different specifications at the analysis level.
Emma: please write this table using R and KableExtra
2 weeks (\textasciitilde{}10 days)
1 month (\textasciitilde{}20 days)
1 semester (\textasciitilde{}100 days)
analysis data
raw data
analysis data
raw data
analysis data
raw data
Scoping
10\% (1 day)
5\% (1 day)
5\% (5 days)
Assessment
35\%
25\%
15\%
Improvement
25\%
0\%
40\%
20\%
30\%
Robustness
25\%
5\%
25\%
25\%
\begin{Shaded}
\begin{Highlighting}[]
\KeywordTok{library}\NormalTok{(tidyverse)}
\KeywordTok{library}\NormalTok{(knitr)}
\KeywordTok{library}\NormalTok{(kableExtra)}
\NormalTok{temp_eval <-}\StringTok{ }\OtherTok{TRUE}
\KeywordTok{options}\NormalTok{(}\DataTypeTok{tinytex.verbose =} \OtherTok{TRUE}\NormalTok{)}
\end{Highlighting}
\end{Shaded}
\hypertarget{assessment}{%
\chapter{Assessment}\label{assessment}}
In this stage, you will review and describe in detail the available reproduction materials, and assess levels of computational reproducibility for the selected outputs, as well as for the overall paper. This stage is designed to record as much of the learning process behind a reproduction as possible to facilitate incremental improvements, and allow future reproducers to pick up easily where others have left off.
First, you will provide a detailed description of the reproduction package. Second, you will connect the outputs you've chosen to reproduce with their corresponding inputs. With these elements in place, you can score the level of reproducibility of each output, and report on paper-level dimensions of reproducibility.
In the \emph{Scoping} stage, you declared a paper, identified the specific claims you will reproduce, and recorded the main estimates that support the claims. In this stage, you will identify all outputs that contain those estimates. You will also decide if you are interested in assessing the reproducibility of that entire output (e.g., ``Table 1''), or will assess only a pre-specified estimates (e.g., ``rows 3 and 4 of Table 1''). Additionally, you can include other outputs of interest.
\textbf{Use \emph{Survey 2} to record your work as part of this step.}
\emph{Tip:} We recommend that you first focus on one specific output (e.g., ``Table 1''). After completing the assessment for this output, you will have a much easier time translating improvements to other outputs.
\hypertarget{describe-inputs}{%
\section{Describe the inputs.}\label{describe-inputs}}
This section explains how to list \emph{all} input materials found or referred to in the reproduction package. First, you will identify data sources and connect them with their raw data files (when available). Second, you will locate and provide a brief description of the analytic data files. Finally, you will locate, inspect, and describe the analytic code used in the paper.
The following terms will be used in this section:
\begin{itemize}
\item
\textbf{Cleaning code:} A script associated primarily with data cleaning. Most of its content is dedicated to actions like deleting variables or observations, merging data sets, removing outliers, or reshaping the structure of the data (from long to wide, or vice versa).
\item
\textbf{Analysis code:} A script associated primarily with analysis. Most of its content is dedicated to actions like running regressions, running hypothesis tests, computing standard errors, and imputing missing values.
\end{itemize}
\hypertarget{desc-sourc}{%
\subsection{Describe the data sources and raw data.}\label{desc-sourc}}
In the paper you chose, find references to all \emph{data sources} used in the analysis. A data source is usually described in narrative form. For example, if in the body of the paper you see text like ``\ldots{}for earnings in 2018 we use the Current Population Survey\ldots{}'', the data source is ``Current Population Survey 2018''. If it is mentioned for the first time on page 1 of the Appendix, its location should be recorded as ``A1''. Do this for all the data sources mentioned in the paper.
Data sources also vary by unit of analysis, with some sources matching the same unit of analysis used in the paper (as in previous examples), while others are less clear (e.g., ``our information on regional minimum wages comes from the Bureau of Labor Statistics.'' This should be recorded as ``regional minimum wages from the Bureau of Labor Statistics'').
Next, look at the reproduction package and map the \emph{data sources} mentioned in the paper to the \emph{data files} in the available materials. Record their folder locations relative to the main reproduction folder\footnote{a relative location takes the form of \texttt{/folder\_in\_rep\_materials/sub\_folder/file.txt}, in contrast to an absolute location that takes the form of \texttt{username/documents/projects/repros/folder\_in\_rep\_materials/sub\_folder/file.txt}}. In addition to looking at the existing data files, we recommend that you review the first lines of all code files (especially cleaning code), looking for lines that call the datasets. Inspecting these scripts may help you understand how different data sources are used, and possibly identify any files that are missing from the reproduction package.
Record this information in this \href{https://docs.google.com/spreadsheets/d/1LUIdVFH0OfR70C7z07TYeE-uWzKI_JIeWUMaYhqEKK0/edit\#gid=0\&range=A1}{standardized spreadsheet} (download it or make a copy for yourself), using the following structure:\\
\begin{table}
\caption{\label{tab:raw-data-information}Raw data information}
\centering
\begin{tabu} to \linewidth {>{\raggedright}X>{\raggedright}X>{\raggedright}X>{\raggedright}X>{\raggedright}X}
\hline
data\_source & page & data\_files & known\_missing & directory\\
\hline
"Current Population Survey 2018" & A1 & cepr\_march\_2018.dta & & /data/\\
\hline
"DHS 2010 - 2013" & 4 & nicaraguaDHS\_2010.csv; boliviaDHS2010.csv; nicaraguaDHS\_2011.csv; nicaraguaDHS\_2012.csv; boliviaDHS\_2012.csv; nicaraguaDHS\_2013.csv; boliviaDHS\_2013.csv & boliviaDHS\_2011.csv & /rawdata/DHS/\\
\hline
"2017 SAT scores" & 4 & Not available & & /data/to\_clean/\\
\hline
... & ... & ... & ... & ...\\
\hline
\end{tabu}
\end{table}
\begin{verbatim}
Raw data information:
|----------------------|------|-----------------------------------------------|---------------------|---------------------|
| data_source | page | data_files | known_missing | directory |
|----------------------|------|-----------------------------------------------|---------------------|---------------------|
| "Current Population | A1 | cepr_march_2018.dta | | \data\ |
| Survey 2018" | | | | |
|----------------------|------|-----------------------------------------------|---------------------|---------------------|
| "DHS 2010 - 2013" | 4 | nicaraguaDHS_2010.csv; | boliviaDHS_2011.csv | \rawdata\DHS\ |
| | | boliviaDHS_2010.csv; nicaraguaDHS_2011.csv; | | |
| | | nicaraguaDHS_2012.csv; boliviaDHS_2012.csv; | | |
| | | nicaraguaDHS_2013.csv; boliviaDHS_2013.csv | | |
|----------------------|------|-----------------------------------------------|---------------------|---------------------|
| "2017 SAT scores" | 4 | Not available | | \data\to_clean\ |
|----------------------|------|-----------------------------------------------|---------------------|---------------------|
| ... | ... | ... | ... | ... |
|----------------------|------|-----------------------------------------------|---------------------|---------------------|
\end{verbatim}
\textbf{Note:} lists if files in the data\_files and known\_missing columns should have entries separated by a semi-colon to for the spreadsheet to be compatible with the ACRE Diagram Builder.
\hypertarget{desc-analy}{%
\subsection{Describe the analytic data sets.}\label{desc-analy}}
List all the analytic files you can find in the reproduction package, and identify their locations relative to the main reproduction folder. Record this information in the \href{https://docs.google.com/spreadsheets/d/1LUIdVFH0OfR70C7z07TYeE-uWzKI_JIeWUMaYhqEKK0/edit\#gid=1299317837\&range=A1}{standardized spreadsheet}.
As you progress through the exercise, add to the spreadsheet a one-line description of each file's main content (for example: \texttt{all\_waves.csv} has the simple description \texttt{data\ for\ region-level\ analysis}). This may be difficult in an initial review, but will become easier as you go along.
The resulting report will have the following structure:
\begin{table}
\caption{\label{tab:analysis-data-information}Analysis data information}
\centering
\begin{tabu} to \linewidth {>{\raggedright}X>{\raggedright}X>{\raggedright}X}
\hline
analysis\_data & location & description\\
\hline
final\_data.csv & /analysis/fig1/ & data for figure1\\
\hline
all\_waves.csv & /final\_data/v1\_april/ & data for region-level analysis\\
\hline
... & ... & ...\\
\hline
\end{tabu}
\end{table}
\begin{verbatim}
Analysis data information:
|----------------|-----------------------|--------------------------------|
| analysis_data | location | description |
|----------------|-----------------------|--------------------------------|
| final_data.csv | /analysis/fig1/ | data for figure1 |
|----------------|-----------------------|--------------------------------|
| all_waves.csv | /final_data/v1_april/ | data for region-level analysis |
|----------------|-----------------------|--------------------------------|
| ... | ... | ... |
|----------------|-----------------------|--------------------------------|
\end{verbatim}
\hypertarget{desc-scripts}{%
\subsection{Describe the code scripts.}\label{desc-scripts}}
List all code files that you found in the reproduction package and identify their locations relative to the master reproduction folder. Review the beginning and end of each code file and identify the inputs required to successfully run the file. Inputs may include data sets or other code scripts that are typically found at the beginning of the script (e.g., \texttt{load}, \texttt{read}, \texttt{source}, \texttt{run}, \texttt{do} ). For each code file, record all inputs together and separate each item with ``;''. Outputs may include other datasets, figures, or plain text files that are typically at the end of a script (e.g., \texttt{save}, \texttt{write}, \texttt{export}). For each code file, record all outputs together and separate each item with ``;''. Provide a one-line description of what each code file does. Record all of this information in the \href{https://docs.google.com/spreadsheets/d/1LUIdVFH0OfR70C7z07TYeE-uWzKI_JIeWUMaYhqEKK0/edit\#gid=1617799822\&range=A1}{standardized spreadsheet}, using the following structure:
\begin{table}
\caption{\label{tab:code-files-information}Code files information}
\centering
\begin{tabu} to \linewidth {>{\raggedright}X>{\raggedright}X>{\raggedright}X>{\raggedright}X>{\raggedright}X>{\raggedright}X}
\hline
file\_name & location & inputs & outputs & description & primary\_type\\
\hline
output\_table1.do & /code/analysis/ & analysis\_data01.csv & output1\_part1.txt & produces first part of table 1 (unformatted) & analysis\\
\hline
data\_cleaning02.R & /code/cleaning & admin\_01raw.csv & analysis\_data02.csv & removes outliers and missing vals from raw admin data & cleaning\\
\hline
... & ... & ... & ... & ... & ...\\
\hline
\end{tabu}
\end{table}
\begin{verbatim}
Code files information:
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
| file_name | location | inputs | outputs | description | primary_type |
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
| output_table1.do | /code/analysis/ | analysis_data01.csv | output1_part1.txt | produces first part | analysis |
| | | | | of table 1 | |
| | | | | (unformatted) | |
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
| data_cleaning02.R | /code/cleaninig/ | admin_01raw.csv | analysis_data02.csv | removes outliers | cleaning |
| | | | | and missing vals | |
| | | | | from raw admin data | |
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
| ... | ... | ... | ... | ... | ... |
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
\end{verbatim}
As you gain an understanding of each code script, you will likely find more inputs and outputs -- we encourage you to update the \href{https://docs.google.com/spreadsheets/d/1LUIdVFH0OfR70C7z07TYeE-uWzKI_JIeWUMaYhqEKK0/edit\#gid=1617799822\&range=A1}{standardized spreadsheet}. Once finished with the reproduction exercise, classify each code file as \emph{analysis} or \emph{cleaning}. We recognize that this may involve subjective judgment, so we suggest that you conduct this classification based on each script's main role.
\textbf{Note:} If a code script takes multiple inputs and/or produces multiple outputs they should be listed as semicolon separated lists in order to be compatible with the ACRE Diagram Builder.
\hypertarget{diagram}{%
\section{Connect each output to all its inputs}\label{diagram}}
Using the information collected above, you can trace your output-to-be-reproduced to its primary sources. Email the standardized spreadsheets from above (sections \ref{desc-sourc}, \ref{desc-analy} and \ref{desc-scripts}) to the ACRE Diagram Builder at \href{mailto:acre@berkeley.edu}{\nolinkurl{acre@berkeley.edu}}. You should receive an email within 24 hours with a reproduction diagram tree that represents the information available on the workflow behind a specific output.
\hypertarget{complete-workflow-information}{%
\subsection{Complete workflow information}\label{complete-workflow-information}}
If you were able to identify all the relevant components in the previous section, the ACRE Diagram Builder will produce a tree diagram that looks similar to the one below.
\begin{verbatim}
table1.tex
|___[code] analysis.R
|___analysis_data.dta
|___[code] final_merge.do
|___cleaned_1_2.dta
| |___[code] clean_merged_1_2.do
| |___merged_1_2.dta
| |___[code] merge_1_2.do
| |___cleaned_1.dta
| | |___[code] clean_raw_1.py
| | |___raw_1.dta
| |___cleaned_2.dta
| |___[code] clean_raw_2.py
| |___raw_2.dta
|___cleaned_3_4.dta
|___[code] clean_merged_3_4.do
|___merged_3_4.dta
|___[code] merge_3_4.do
|___cleaned_3.dta
| |___[code] clean_raw_3.py
| |___raw_3.dta
|___cleaned_4.dta
|___[code] clean_raw_4.py
|___raw_4.dta
\end{verbatim}
This diagram, built with the information you provided, is already an important contribution to understanding the necessary components required to reproduce a specific output. It summarizes key information to allow for more constructive exchanges with original authors or other reproducers. For example, when contacting the authors for guidance, you can use the diagram to point out specific files you need. Formulating your request this way makes it easier for authors to respond and demonstrates that you have a good understanding of the reproduction package.
\hypertarget{incomplete-workflow-information}{%
\subsection{Incomplete workflow information}\label{incomplete-workflow-information}}
In many cases, some of the components of the workflow will not be easily identifiable (or missing) in the reproduction package. Here the Diagram Builder will return a partial reproduction tree diagram. For example, if the files \texttt{merge\_1\_2.do}, \texttt{merge\_3\_4.do}, and \texttt{final\_merge.do} are missing from the previous diagram, the ACRE Diagram Builder will produce the following diagram:
\begin{verbatim}
cleaned_3.dta
|___[code] clean_raw_3.py
|___raw_3.dta
table1.tex
|___[code] analysis.R
|___analysis_data.dta
cleaned_3_4.dta
|___[code] clean_merged_3_4.do
|___merged_3_4.dta
cleaned_1.dta
|___[code] clean_raw_1.py
|___raw_1.dta
cleaned_2.dta
|___[code] clean_raw_2.py
|___raw_2.dta
cleaned_4.dta
|___[code] clean_raw_4.py
|___raw_4.dta
cleaned_1_2.dta
|___[code] clean_merged_1_2.do
|___merged_1_2.dta
Unused data sources: None.
\end{verbatim}
In this case, you can still manually combine this partial information with your knowledge from the paper and own judgement to produce a ``candidate'' tree diagram (which might lead to different reproducers recreating different diagrams). This may look like the following:
\begin{verbatim}
table1.tex
|___[code] analysis.R
|___analysis_data.dta
|___MISSSING CODE FILE(S) #3
|___cleaned_3_4.dta
| |___[code] clean_merged_3_4.do
| |___merged_3_4.dta
| |___MISSSING CODE FILE(S) #2
| |___cleaned_3.dta
| | |___[code] clean_raw_3.py
| | |___raw_3.dta
| |___cleaned_4.dta
| |___[code] clean_raw_4.py
| |___raw_4.dta
|___cleaned_1_2.dta
|___[code] clean_merged_1_2.do
|___merged_1_2.dta
|___MISSSING CODE FILE(S) #1
|___cleaned_1.dta
| |___[code] clean_raw_1.py
| |___raw_1.dta
|
|___cleaned_2.dta
|___[code] clean_raw_2.py
|___raw_2.dta
\end{verbatim}
To leave a record of the reconstructed diagrams, you will have to amend the input spreadsheets using placeholders for the missing components. In the example above, you should add the following entries to the code description spreadsheet:
\begin{table}
\caption{\label{tab:adding-rows}Adding rows to code spreadsheet}
\centering
\begin{tabu} to \linewidth {>{\raggedright}X>{\raggedright}X>{\raggedright}X>{\raggedright}X>{\raggedright}X>{\raggedright}X}
\hline
file\_name & location & inputs & outputs & description & primary\_type\\
\hline
... & ... & ... & ... & ... & ...\\
\hline
missing\_file1 & unknown & cleaned\_1.dta, cleaned\_2.dta & merged\_1\_2.dta & missing code & unknown\\
\hline
missing\_file2 & unknown & cleaned\_3.dta, cleaned\_4.dta & merged\_3\_4.dta & missing code & unknown\\
\hline
missing\_file3 & unknown & merged\_3\_4.dta, merged\_1\_2.dta & analysis\_data.dta & missing code & unknown\\
\hline
\end{tabu}
\end{table}
\begin{verbatim}
Adding rows to code spreadsheet:
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
| file_name | location | inputs | outputs | description | primary_type |
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
| ... | ... | ... | ... | ... | ... |
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
| missing_file1 | unknown | cleaned_1.dta; | merged_1_2.dta | missing code | unknown |
| | | cleaned_2.dta | | | |
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
| missing_file2 | unknown | cleaned_3.dta; | merged_3_4.dta | missing code | unknown |
| | | cleaned_4.dta | | | |
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
| missing_file3 | unknown | merged_3_4.dta; | analysis_data.dta | missing code | unknown |
| | | merged_1_2.dta | | | |
|-------------------|------------------|---------------------|---------------------|----------------------|--------------|
\end{verbatim}
As in the cases with complete workflows, these diagrams (fragmented or reconstructed trees) provide important information for assessing and improving the reproducibility of specific outputs. Reproducers can compare reconstructed trees and/or contact original authors with highly specific inquiries.
For more examples of diagrams connecting final outputs to initial raw data, \protect\hyperlink{additional-diagrams}{see here}.
\hypertarget{unused-data-sources}{%
\subsection{Unused Data Sources}\label{unused-data-sources}}
It is possible that not all data included in a replication package are actually used in code scripts in the reproduction package. This would be the case if, for example, the raw data and analysis data are included, but not the script that generates the analysis data. As a concrete example, consider what the original diagram above would look like if the only code included in the reproduction package were analysis.R:
\begin{verbatim}
table1.tex
|___[code] analysis.R
|___analysis_data.dta
Unused data sources:
raw_1.dta
raw_2.dta
raw_3.dta
raw_4.dta
Unused analysis data:
cleaned_1.dta
cleaned_2.dta
cleaned_3.dta
cleaned_4.dta
merged_1_2.dta
merged_3_4.dta
cleaned_1_2.dta
cleaned_3_4.dta
\end{verbatim}
In this case, there are many data files that were listed in the raw data and analytic data spreadsheets that are not used by any code script in the replication package.
\hypertarget{score}{%
\section{Assign a reproducibility score.}\label{score}}
Once you have identified all possible inputs and have a clear understanding of the connection between the outputs and inputs, you can start to assess the output-specific level of reproducibility.
Take note of the following concepts in this section:
\begin{itemize}
\item
\textbf{Computationally Reproducible from Analytic data (CRA):} The output can be reproduced with minimal effort starting from the \emph{analytic} datasets.
\item
\textbf{Computationally Reproducible from Raw data (CRR):} The output can be reproduced with minimal effort from the \emph{raw} datasets.
\item
\textbf{Minimal effort:} One hour or less is required to run the code, not including computing time.
\end{itemize}
\hypertarget{levels-of-computational-reproducibility-for-a-specific-output}{%
\subsection{Levels of Computational Reproducibility for a Specific Output}\label{levels-of-computational-reproducibility-for-a-specific-output}}
Each level of computational reproducibility is defined by the availability of data and materials, and whether or not the available materials faithfully reproduce the output of interest. The description of each level also includes possible improvements that can help advance the reproducibility of the output to a higher level. You will learn in more detail about the possible improvements.
Note that the assessment is made \emph{at the output level} -- a paper can be highly reproducible for its main results, but suffer from low reproducibility for other outputs. The assessment includes a 10-point scale, where 1 represents that, under current circumstances, reproducers cannot access any reproduction package, while 10 represents access to all the materials and being able to reproduce the target outcome from the raw data.
\begin{itemize}
\tightlist
\item
\textbf{Level 1 (L1):} No data or code are available. Possible improvements include adding: raw data (+AD), analysis data (+RD), cleaning code (+CC), and analysis code (+AC).
\end{itemize}
You will have detected papers that are reproducible at Level 1 as part of the Scoping stage (unsuccessful candidate papers). Make sure to take record them in Survey 1.
\begin{itemize}
\item
\textbf{Level 2 (L2):} Code scripts are available (partial or complete), but no data are available. Possible improvements include adding: raw data (+AD) and analysis data (+RD).
\item
\textbf{Level 3 (L3):} Analytic data and code are partially available, but raw data and cleaning code are not. Possible improvements include: completing analysis data and/or code, adding raw data (+RD), and adding analysis code (+AC).
\item
\textbf{Level 4 (L4):} All analytic data sets and analysis code are available, but code does not run or produces results different than those in the paper (not CRA). Possible improvements include: debugging the analysis code (DAC) or obtaining raw data (+RD).
\item
\textbf{Level 5 (L5):} Analytic data sets and analysis code are available. They produce the same results as presented in the paper (CRA). The reproducibility package may be improved by obtaining the original raw data sets.
\end{itemize}
This is the highest level that most published research papers can attain currently. Computational reproducibility \emph{from raw data} is required for papers that are reproducible at Level 6 and above.
\begin{itemize}
\item
\textbf{Level 6 (L6):} Cleaning code is partially available, but raw data is not. Possible improvements include: completing cleaning code (+CC) and/or raw data (+RD).
\item
\textbf{Level 7 (L7):} Cleaning code is available and complete, but raw data is not. Possible improvements include: adding raw data (+RD).
\item
\textbf{Level 8 (L8):} Cleaning code is available and complete, and raw data is partially available. Possible improvements include: adding raw data (+RD).
\item
\textbf{Level 9 (L9):} All the materials (raw data, analytic data, cleaning code, and analysis code) are available. The analysis code produces the same output as presented in the paper (CRA). However, the cleaning code does not run or produces different results that those presented in the paper (not CRR). Possible improvements include: debugging the cleaning code (DCC).
\item
\textbf{Level 10 (L10):} All the materials are available and produce the same results as presented in the paper with minimal effort, starting from the analytic data (yes CRA) or the raw data (yes CRR). Note that Level 10 is aspirational and may be very difficult to attain for most research published today.
\end{itemize}
The following figure summarizes the different levels of computational reproducibility (for any given output). For each level, there will be improvements that have been made (\texttt{✔}) or can be made to move up one level of reproducibility (-).
\begin{table}
\caption{\label{tab:levels-of-computational-reproducibility}Levels of Computational Reproducibility \
(P denotes "partial", C denotes "complete")}
\centering
\begin{tabular}[t]{l|l|l|l|l|l|l|l|l|l|l}
\hline
\multicolumn{1}{c|}{ } & \multicolumn{10}{c}{Availability of materials, and reproducibility} \\
\cline{2-11}
\multicolumn{1}{c|}{ } & \multicolumn{2}{c|}{Analysis Code} & \multicolumn{2}{c|}{Analysis Data} & \multicolumn{1}{c|}{CRA} & \multicolumn{2}{c|}{Cleaning Code} & \multicolumn{2}{c|}{Raw Data} & \multicolumn{1}{c}{CRR} \\
\cline{2-3} \cline{4-5} \cline{6-6} \cline{7-8} \cline{9-10} \cline{11-11}
& P & C & P & C & & P & C & P & C & \\
\hline
L1: No materials & -- & -- & -- & -- & -- & -- & -- & -- & -- & --\\
\hline
L2: Only code & ✔ & ✔ & -- & -- & -- & -- & -- & -- & -- & --\\
\hline
L3: Partial analysis data \& code & ✔ & ✔ & ✔ & -- & -- & -- & -- & -- & -- & --\\
\hline
L4: All analysis data \& code & ✔ & ✔ & ✔ & ✔ & -- & -- & -- & -- & -- & --\\
\hline
L5: Reproducible from analysis & ✔ & ✔ & ✔ & ✔ & ✔ & -- & -- & -- & -- & --\\
\hline
L6: Some cleaning code & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & -- & -- & -- & --\\
\hline
L7: All cleaning code & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & -- & -- & --\\
\hline
L8: Some raw data & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & -- & --\\
\hline
L9: All raw data & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & --\\
\hline
L10: Reproducible from raw data & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & ✔ & ✔\\
\hline
\end{tabular}
\end{table}
\begin{verbatim}
Levels of Computational Reproducibility
(P denotes "partial", C denotes "complete")
| Availability of materials, and reproducibility |
|------------------------------------------------|
|Analysis| Analysis| | Cleaning| Raw | |
|Code | Data | CRA | Code | Data | CRR |
| P | C | P | C | | P | C | P | C | |
---------|---------|-----|---------|-------|-----|
L1: No materials.................| - - | - - | - | - - | - - | - |
---------------------------------|--------|---------|-----|---------|-------|-----|
L2: Only code ...................| ✔ ✔ | - - | - | - - | - - | - |
L3: Partial analysis data & code.| ✔ ✔ | ✔ - | - | - - | - - | - |
L4: All analysis data & code.....| ✔ ✔ | ✔ ✔ | - | - - | - - | - |
L5: Reproducible from analysis...| ✔ ✔ | ✔ ✔ | ✔ | - - | - - | - |
---------------------------------|--------|---------|-----|---------|-------|-----|
L6: Some cleaning code...........| ✔ ✔ | ✔ ✔ | ✔ | ✔ - | - - | - |
L7: All cleaning code............| ✔ ✔ | ✔ ✔ | ✔ | ✔ ✔ | - - | - |
L8: Some raw data................| ✔ ✔ | ✔ ✔ | ✔ | ✔ ✔ | ✔ - | - |
L9: All raw data.................| ✔ ✔ | ✔ ✔ | ✔ | ✔ ✔ | ✔ ✔ | - |
L10:Reproducible from raw data...| ✔ ✔ | ✔ ✔ | ✔ | ✔ ✔ | ✔ ✔ | ✔ |
\end{verbatim}
You may disagree with some of the levels outlined above, particularly wherever subjective judgment may be required. If so, you are welcome to interpret the levels as unordered categories (independent from their sequence) and suggest improvements using the ``Edit'' button above (top left corner if you are reading this document in your browser).
\hypertarget{adjusting-levels-to-account-for-confidentialproprietary-data}{%
\subsubsection*{Adjusting Levels To Account for Confidential/Proprietary Data}\label{adjusting-levels-to-account-for-confidentialproprietary-data}}
\addcontentsline{toc}{subsubsection}{Adjusting Levels To Account for Confidential/Proprietary Data}
A large portion of published research in economics uses confidential or proprietary data, most often government data from tax records or service provision and what is generally referred to as \emph{administrative data}. Since administrative and proprietary data are rarely publicly accessible, some of the reproducibility levels presented above only apply once modified. The underlying theme of these modifications is that when data cannot be provided, you can assign a reproducibility score based on the level of detail in the instructions for accessing the data. Similarly, when reproducibility cannot be verified based on publicly available materials, the reproduction materials should demonstrate that a competent and unbiased third party (not involved in the original research team) has been able to reproduce the results.
\begin{itemize}
\item
\textbf{Levels 1 and 2} can be applied as described above.
\item
\textbf{Adjusted Level 3 (L3*):} All analysis code is provided, but only partial instructions on how to access the \textbf{\emph{analysis data}} are available. This means that the authors have provided some, but not all, of the following information:
\begin{enumerate}
\def\labelenumi{\alph{enumi}.}
\tightlist
\item
\emph{Contact information}, including name of the organization(s) that provides access to the data and contact information of at least one individual.
\item
\emph{Terms of use}, including licenses and eligibility criteria for accessing the data, if any.
\item
\emph{Information on data files (meta-data)}, including the name(s) and number of files, file size(s), relevant file version(s), and number of variables and observations in each file. Though not required, other relevant information may be included, including a description dataset dictionary, summary statistics, and synthetic data (fake data with the same statistical properties as the original data)
\item
\emph{Estimated costs for access}, including monetary costs such as fees and licences required to access the data, and non-monetary costs such as wait times and specific geographical locations from where researchers need to access the data.
\end{enumerate}
\item
\textbf{Adjusted Level 4 (L4*):} All analysis code is provided, and complete and detailed instructions on how to access the \emph{analysis data} are available.
\item
\textbf{Adjusted Level 5 (L5*):} All requirements for Level 4* are met, and the authors provide a certification that the output can be reproduced from the analysis data (CRA) by a third party. Examples include a signed letter by a disinterested reproducer or an official reproducibility certificate from a certification agency for data and code (e.g., see \href{https://www.cascad.tech/}{cascad}).
\item
\textbf{Levels 6 and 7} can be applied as described above.
\item
\textbf{Adjusted Level 8 (L8*):} All requirements for Level 7* are met, but instructions for accessing the \textbf{\emph{raw data}} are incomplete. Use the instructions described in Level 3 above to assess the instructions' completeness.
\item
\textbf{Adjusted Level 9 (L9*):} All requirements for Level 8* are met, and instructions for accessing the \textbf{\emph{raw data}} are complete.