project.yaml

version: '3.0'

expectations:
  population_size: 10000

actions:

## 1. EXTRACT DATA FOR EXPOSED
  generate_community_cases_cohort:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_covid_communitycases
    outputs:
      highly_sensitive:
        cohort: output/input_covid_communitycases.csv


# 2. EXTRACT DATA FOR POTENTIAL COMPARATORS
# new extract potential comparators code... I guess I'll have to this twice for each control group
# or potentially 3 times: 2018 matches, 2019 matches, contemporary matches?

 # *** contemporary potential comparators
  generate_potential_controls_contemporary:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_controls_contemporary
    outputs:
      highly_sensitive:
        cohort: output/input_controls_contemporary.csv


# 2b. CHECK NUMBERS IN EACH STP PRIOR TO SPLITTING AND APPENDING STEPS BELOW

 # *** contemporary potential comparators
  checkSTPs_in_original_extract:
    run: stata-mp:latest analysis/00_checkSTPs_in_original_extract_an.do
    needs: [generate_community_cases_cohort, generate_potential_controls_contemporary]
    outputs:
      moderately_sensitive:
        log: logs/00_checkSTPs_in_original_extract_an.log



# 3. SPLIT CASES AND COMPARATORS UP BY STP
  01_longCovidSymp_split_by_stp:
    run: stata-mp:latest analysis/01_longCovidSymp_cr_split_by_stp.do
    needs: [generate_community_cases_cohort, generate_potential_controls_contemporary]
    outputs:
      highly_sensitive:
        case_stp5: output/input_controls_contemporary_stp5.csv
        case_stp6: output/input_controls_contemporary_stp6.csv
        case_stp7: output/input_controls_contemporary_stp7.csv
        case_stp8: output/input_controls_contemporary_stp8.csv
        case_stp9: output/input_controls_contemporary_stp9.csv
        case_stp10: output/input_controls_contemporary_stp10.csv
        case_stp12: output/input_controls_contemporary_stp12.csv
        case_stp13: output/input_controls_contemporary_stp13.csv
        case_stp14: output/input_controls_contemporary_stp14.csv
        case_stp15: output/input_controls_contemporary_stp15.csv
        case_stp16: output/input_controls_contemporary_stp16.csv
        case_stp17: output/input_controls_contemporary_stp17.csv
        case_stp20: output/input_controls_contemporary_stp20.csv
        case_stp21: output/input_controls_contemporary_stp21.csv
        case_stp22: output/input_controls_contemporary_stp22.csv
        case_stp23: output/input_controls_contemporary_stp23.csv
        case_stp24: output/input_controls_contemporary_stp24.csv
        case_stp25: output/input_controls_contemporary_stp25.csv
        case_stp26: output/input_controls_contemporary_stp26.csv
        case_stp27: output/input_controls_contemporary_stp27.csv
        case_stp29: output/input_controls_contemporary_stp29.csv
        case_stp33: output/input_controls_contemporary_stp33.csv
        case_stp35: output/input_controls_contemporary_stp35.csv
        case_stp36: output/input_controls_contemporary_stp36.csv
        case_stp37: output/input_controls_contemporary_stp37.csv
        case_stp40: output/input_controls_contemporary_stp40.csv
        case_stp41: output/input_controls_contemporary_stp41.csv
        case_stp42: output/input_controls_contemporary_stp42.csv
        case_stp43: output/input_controls_contemporary_stp43.csv
        case_stp44: output/input_controls_contemporary_stp44.csv
        case_stp49: output/input_controls_contemporary_stp49.csv
        control_stp5: output/input_covid_communitycases_stp5.csv
        control_stp6: output/input_covid_communitycases_stp6.csv
        control_stp7: output/input_covid_communitycases_stp7.csv
        control_stp8: output/input_covid_communitycases_stp8.csv
        control_stp9: output/input_covid_communitycases_stp9.csv
        control_stp10: output/input_covid_communitycases_stp10.csv
        control_stp12: output/input_covid_communitycases_stp12.csv
        control_stp13: output/input_covid_communitycases_stp13.csv
        control_stp14: output/input_covid_communitycases_stp14.csv
        control_stp15: output/input_covid_communitycases_stp15.csv
        control_stp16: output/input_covid_communitycases_stp16.csv
        control_stp17: output/input_covid_communitycases_stp17.csv
        control_stp20: output/input_covid_communitycases_stp20.csv
        control_stp21: output/input_covid_communitycases_stp21.csv
        control_stp22: output/input_covid_communitycases_stp22.csv
        control_stp23: output/input_covid_communitycases_stp23.csv
        control_stp24: output/input_covid_communitycases_stp24.csv
        control_stp25: output/input_covid_communitycases_stp25.csv
        control_stp26: output/input_covid_communitycases_stp26.csv
        control_stp27: output/input_covid_communitycases_stp27.csv
        control_stp29: output/input_covid_communitycases_stp29.csv
        control_stp33: output/input_covid_communitycases_stp33.csv
        control_stp35: output/input_covid_communitycases_stp35.csv
        control_stp36: output/input_covid_communitycases_stp36.csv
        control_stp37: output/input_covid_communitycases_stp37.csv
        control_stp40: output/input_covid_communitycases_stp40.csv
        control_stp41: output/input_covid_communitycases_stp41.csv
        control_stp42: output/input_covid_communitycases_stp42.csv
        control_stp43: output/input_covid_communitycases_stp43.csv
        control_stp44: output/input_covid_communitycases_stp44.csv
        control_stp49: output/input_covid_communitycases_stp49.csv
      moderately_sensitive:
        log: logs/01_longCovidSymp_split_by_stp.log



# 4. MATCH CASES TO COMPARATORS
  # *** contemporary potential comparators for each stp
  match_contemporary_stp5:
    run: python:latest python analysis/match_contemporary_stp5.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp5: output/matched_cases_stp5.csv
        matched_matches_stp5: output/matched_matches_stp5.csv
        matched_all_stp5: output/matched_combined_stp5.csv
      moderately_sensitive:
        matching_report_stp5: output/matching_report_stp5.txt

  match_contemporary_stp6:
    run: python:latest python analysis/match_contemporary_stp6.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp6: output/matched_cases_stp6.csv
        matched_matches_stp6: output/matched_matches_stp6.csv
        matched_all_stp6: output/matched_combined_stp6.csv
      moderately_sensitive:
         matching_report_stp6: output/matching_report_stp6.txt

  match_contemporary_stp7:
    run: python:latest python analysis/match_contemporary_stp7.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp7: output/matched_cases_stp7.csv
        matched_matches_stp7: output/matched_matches_stp7.csv
        matched_all_stp7: output/matched_combined_stp7.csv
      moderately_sensitive:
         matching_report_stp7: output/matching_report_stp7.txt

  match_contemporary_stp8:
    run: python:latest python analysis/match_contemporary_stp8.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp8: output/matched_cases_stp8.csv
        matched_matches_stp8: output/matched_matches_stp8.csv
        matched_all_stp8: output/matched_combined_stp8.csv
      moderately_sensitive:
         matching_report_stp8: output/matching_report_stp8.txt

  match_contemporary_stp9:
    run: python:latest python analysis/match_contemporary_stp9.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp9: output/matched_cases_stp9.csv
        matched_matches_stp9: output/matched_matches_stp9.csv
        matched_all_stp9: output/matched_combined_stp9.csv
      moderately_sensitive:
         matching_report_stp9: output/matching_report_stp9.txt

  match_contemporary_stp10:
    run: python:latest python analysis/match_contemporary_stp10.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp10: output/matched_cases_stp10.csv
        matched_matches_stp10: output/matched_matches_stp10.csv
        matched_all_stp10: output/matched_combined_stp10.csv
      moderately_sensitive:
        matching_report_stp10: output/matching_report_stp10.txt
      
  match_contemporary_stp12:
    run: python:latest python analysis/match_contemporary_stp12.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp12: output/matched_cases_stp12.csv
        matched_matches_stp12: output/matched_matches_stp12.csv
        matched_all_stp12: output/matched_combined_stp12.csv
      moderately_sensitive:
        matching_report_stp12: output/matching_report_stp12.txt

  match_contemporary_stp13:
    run: python:latest python analysis/match_contemporary_stp13.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp13: output/matched_cases_stp13.csv
        matched_matches_stp13: output/matched_matches_stp13.csv
        matched_all_stp13: output/matched_combined_stp13.csv
      moderately_sensitive:
        matching_report_stp13: output/matching_report_stp13.txt

  match_contemporary_stp14:
    run: python:latest python analysis/match_contemporary_stp14.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp14: output/matched_cases_stp14.csv
        matched_matches_stp14: output/matched_matches_stp14.csv
        matched_all_stp14: output/matched_combined_stp14.csv
      moderately_sensitive:
        matching_report_stp14: output/matching_report_stp14.txt

  match_contemporary_stp15:
    run: python:latest python analysis/match_contemporary_stp15.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp15: output/matched_cases_stp15.csv
        matched_matches_stp15: output/matched_matches_stp15.csv
        matched_all_stp15: output/matched_combined_stp15.csv
      moderately_sensitive:
        matching_report_stp15: output/matching_report_stp15.txt
       
  match_contemporary_stp16:
    run: python:latest python analysis/match_contemporary_stp16.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp16: output/matched_cases_stp16.csv
        matched_matches_stp16: output/matched_matches_stp16.csv
        matched_all_stp16: output/matched_combined_stp16.csv
      moderately_sensitive:
        matching_report_stp16: output/matching_report_stp16.txt
        
  match_contemporary_stp17:
    run: python:latest python analysis/match_contemporary_stp17.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp17: output/matched_cases_stp17.csv
        matched_matches_stp17: output/matched_matches_stp17.csv
        matched_all_stp17: output/matched_combined_stp17.csv
      moderately_sensitive:
        matching_report_stp17: output/matching_report_stp17.txt
        
  match_contemporary_stp20:
    run: python:latest python analysis/match_contemporary_stp20.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp20: output/matched_cases_stp20.csv
        matched_matches_stp20: output/matched_matches_stp20.csv
        matched_all_stp20: output/matched_combined_stp20.csv
      moderately_sensitive:
        matching_report_stp20: output/matching_report_stp20.txt

  match_contemporary_stp21:
    run: python:latest python analysis/match_contemporary_stp21.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp21: output/matched_cases_stp21.csv
        matched_matches_stp21: output/matched_matches_stp21.csv
        matched_all_stp21: output/matched_combined_stp21.csv
      moderately_sensitive:
        matching_report_stp21: output/matching_report_stp21.txt
       
  match_contemporary_stp22:
    run: python:latest python analysis/match_contemporary_stp22.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp22: output/matched_cases_stp22.csv
        matched_matches_stp22: output/matched_matches_stp22.csv
        matched_all_stp22: output/matched_combined_stp22.csv
      moderately_sensitive:
        matching_report_stp22: output/matching_report_stp22.txt

  match_contemporary_stp23:
    run: python:latest python analysis/match_contemporary_stp23.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp23: output/matched_cases_stp23.csv
        matched_matches_stp23: output/matched_matches_stp23.csv
        matched_all_stp23: output/matched_combined_stp23.csv
      moderately_sensitive:
        matching_report_stp23: output/matching_report_stp23.txt

  match_contemporary_stp24:
    run: python:latest python analysis/match_contemporary_stp24.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp24: output/matched_cases_stp24.csv
        matched_matches_stp24: output/matched_matches_stp24.csv
        matched_all_stp24: output/matched_combined_stp24.csv
      moderately_sensitive:
        matching_report_stp24: output/matching_report_stp24.txt

  match_contemporary_stp25:
    run: python:latest python analysis/match_contemporary_stp25.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp25: output/matched_cases_stp25.csv
        matched_matches_stp25: output/matched_matches_stp25.csv
        matched_all_stp25: output/matched_combined_stp25.csv
      moderately_sensitive:
        matching_report_stp25: output/matching_report_stp25.txt

  match_contemporary_stp26:
    run: python:latest python analysis/match_contemporary_stp26.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp26: output/matched_cases_stp26.csv
        matched_matches_stp26: output/matched_matches_stp26.csv
        matched_all_stp26: output/matched_combined_stp26.csv
      moderately_sensitive:
        matching_report_stp26: output/matching_report_stp26.txt
        
  match_contemporary_stp27:
    run: python:latest python analysis/match_contemporary_stp27.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp27: output/matched_cases_stp27.csv
        matched_matches_stp27: output/matched_matches_stp27.csv
        matched_all_stp27: output/matched_combined_stp27.csv
      moderately_sensitive:
        matching_report_stp27: output/matching_report_stp27.txt

  match_contemporary_stp29:
    run: python:latest python analysis/match_contemporary_stp29.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp29: output/matched_cases_stp29.csv
        matched_matches_stp29: output/matched_matches_stp29.csv
        matched_all_stp29: output/matched_combined_stp29.csv
      moderately_sensitive:
        matching_report_stp29: output/matching_report_stp29.txt

  match_contemporary_stp33:
    run: python:latest python analysis/match_contemporary_stp33.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp33: output/matched_cases_stp33.csv
        matched_matches_stp33: output/matched_matches_stp33.csv
        matched_all_stp33: output/matched_combined_stp33.csv
      moderately_sensitive:
        matching_report_stp33: output/matching_report_stp33.txt

  match_contemporary_stp35:
    run: python:latest python analysis/match_contemporary_stp35.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp35: output/matched_cases_stp35.csv
        matched_matches_stp35: output/matched_matches_stp35.csv
        matched_all_stp35: output/matched_combined_stp35.csv
      moderately_sensitive:
        matching_report_stp35: output/matching_report_stp35.txt

  match_contemporary_stp36:
    run: python:latest python analysis/match_contemporary_stp36.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp36: output/matched_cases_stp36.csv
        matched_matches_stp36: output/matched_matches_stp36.csv
        matched_all_stp36: output/matched_combined_stp36.csv
      moderately_sensitive:
        matching_report_stp36: output/matching_report_stp36.txt

  match_contemporary_stp37:
    run: python:latest python analysis/match_contemporary_stp37.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp37: output/matched_cases_stp37.csv
        matched_matches_stp37: output/matched_matches_stp37.csv
        matched_all_stp37: output/matched_combined_stp37.csv
      moderately_sensitive:
        matching_report_stp37: output/matching_report_stp37.txt

  match_contemporary_stp40:
    run: python:latest python analysis/match_contemporary_stp40.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp40: output/matched_cases_stp40.csv
        matched_matches_stp40: output/matched_matches_stp40.csv
        matched_all_stp40: output/matched_combined_stp40.csv
      moderately_sensitive:
        matching_report_stp40: output/matching_report_stp40.txt

  match_contemporary_stp41:
    run: python:latest python analysis/match_contemporary_stp41.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp41: output/matched_cases_stp41.csv
        matched_matches_stp41: output/matched_matches_stp41.csv
        matched_all_stp41: output/matched_combined_stp41.csv
      moderately_sensitive:
        matching_report_stp41: output/matching_report_stp41.txt

  match_contemporary_stp42:
    run: python:latest python analysis/match_contemporary_stp42.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp42: output/matched_cases_stp42.csv
        matched_matches_stp42: output/matched_matches_stp42.csv
        matched_all_stp42: output/matched_combined_stp42.csv
      moderately_sensitive:
        matching_report_stp41: output/matching_report_stp42.txt

  match_contemporary_stp43:
    run: python:latest python analysis/match_contemporary_stp43.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp43: output/matched_cases_stp43.csv
        matched_matches_stp43: output/matched_matches_stp43.csv
        matched_all_stp43: output/matched_combined_stp43.csv
      moderately_sensitive:
         matching_report_stp43: output/matching_report_stp43.txt

  match_contemporary_stp44:
    run: python:latest python analysis/match_contemporary_stp44.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp44: output/matched_cases_stp44.csv
        matched_matches_stp44: output/matched_matches_stp44.csv
        matched_all_stp44: output/matched_combined_stp44.csv
      moderately_sensitive:
         matching_report_stp44: output/matching_report_stp44.txt

  match_contemporary_stp49:
    run: python:latest python analysis/match_contemporary_stp49.py
    needs: [01_longCovidSymp_split_by_stp]
    outputs:
      highly_sensitive:
        matched_cases_stp49: output/matched_cases_stp49.csv
        matched_matches_stp49: output/matched_matches_stp49.csv
        matched_all_stp49: output/matched_combined_stp49.csv
      moderately_sensitive:
         matching_report_stp49: output/matching_report_stp49.txt


# 5. CHECK VARIABLES IN MATCHED FILES
  # *** contemporary potential comparators for each stp
  checkVars_inMatchedFiles:
    run: stata-mp:latest analysis/05_check_vars_after_matching.do
    needs: [match_contemporary_stp5]
    outputs:
      moderately_sensitive:
        log: logs/05_check_vars_after_matching.log



# 6. COMBINE MATCHED CASES AND MATCHED COMPARATORS BY STP
  combine_matched_CASESstps:
    run: stata-mp:latest analysis/06a_longCovidSymp_cr_combine_CASEstps_after_matching.do
    needs: [match_contemporary_stp5, match_contemporary_stp6, match_contemporary_stp7, match_contemporary_stp8, match_contemporary_stp9,
    match_contemporary_stp10, match_contemporary_stp12, match_contemporary_stp13, match_contemporary_stp14, match_contemporary_stp15,
    match_contemporary_stp16, match_contemporary_stp17, match_contemporary_stp20, match_contemporary_stp21, match_contemporary_stp22,
    match_contemporary_stp23, match_contemporary_stp24, match_contemporary_stp25, match_contemporary_stp26, match_contemporary_stp27,
    match_contemporary_stp29, match_contemporary_stp33, match_contemporary_stp35, match_contemporary_stp36, match_contemporary_stp37, 
    match_contemporary_stp40, match_contemporary_stp41, match_contemporary_stp42, match_contemporary_stp43, match_contemporary_stp44, 
    match_contemporary_stp49]
    outputs:
      highly_sensitive:
        matched_cases_allstps: output/input_covid_matched_cases_allSTPs.csv
      moderately_sensitive:
        log: logs/06a_longCovidSymp_cr_combine_CASEstps_after_matching.log


  combine_matched_COMPARATORstps:
    run: stata-mp:latest analysis/06b_longCovidSymp_cr_combine_COMPARATORstps_after_matching.do
    needs: [match_contemporary_stp5, match_contemporary_stp6, match_contemporary_stp7, match_contemporary_stp8, match_contemporary_stp9,
    match_contemporary_stp10, match_contemporary_stp12, match_contemporary_stp13, match_contemporary_stp14, match_contemporary_stp15,
    match_contemporary_stp16, match_contemporary_stp17, match_contemporary_stp20, match_contemporary_stp21, match_contemporary_stp22,
    match_contemporary_stp23, match_contemporary_stp24, match_contemporary_stp25, match_contemporary_stp26, match_contemporary_stp27,
    match_contemporary_stp29, match_contemporary_stp33, match_contemporary_stp35, match_contemporary_stp36, match_contemporary_stp37, 
    match_contemporary_stp40, match_contemporary_stp41, match_contemporary_stp42, match_contemporary_stp43, match_contemporary_stp44, 
    match_contemporary_stp49]
    outputs:
      highly_sensitive:
        matched_matches_allstps: output/input_covid_matched_matches_allSTPs.csv
      moderately_sensitive:
        log: logs/06b_longCovidSymp_cr_combine_COMPARATORstps_after_matching.log



 # 7. ADD FURTHER VARIABLES TO CONTROLS FOLLOWING MATCHING
  add_allVars_contemporary_controls:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_complete_controls_contemporary 
    needs: [combine_matched_CASESstps]
    outputs:
      highly_sensitive:
        matched_controls_contemporary_wVars: output/input_complete_controls_contemporary.csv



# 8. ADD FURTHER VARIABLES TO CASES FOLLOWING MATCHING
  add_allVars_cases:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_complete_covid_communitycases 
    needs: [combine_matched_COMPARATORstps]
    outputs:
      highly_sensitive:
        matched_cases_contemporary_wVars: output/input_complete_covid_communitycases.csv



# 9. CREATE ANALYSIS FILE (ALSO COMBINES CASES AND CONTROLS)
  create_analysis_file:
    run: stata-mp:latest analysis/09_longCovidSymp_cr_combined_cases_contemporary_controls.do 
    needs: [add_allVars_contemporary_controls, add_allVars_cases]
    outputs:
      highly_sensitive:
        # casesContemporary: output/cases_contemporary.dta 
        # controlsContemporary: output/controls_contemporary.dta 
        mainAnalysisFile: output/longCovidSymp_analysis_dataset_contemporary.dta 
        unmatchedCasesAnalysisFile: output/longCovidSymp_UnmatchedCases_analysis_dataset.dta
      moderately_sensitive:
        log: logs/09_longCovidSymp_cr_combined_cases_controls.log