project.yaml

version: '3.0'

expectations:
  population_size: 1000

actions:
  # Generating static variables 
  generate_study_population_static:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_static --output-format=csv
    outputs:
      highly_sensitive:
        cohort: output/input_static.csv

  # Generating static variables 
  generate_study_population_static_2020:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_static_2020 --output-format=csv
    outputs:
      highly_sensitive:
        cohort: output/input_static_2020.csv
  # Generates study population for CV - start date = 2018-03-01
  generate_study_population:
    run: cohortextractor:latest generate_cohort 
      --study-definition study_definition 
      --index-date-range "2018-03-01 to 2022-04-30 by month" 
      --output-dir=output/measures 
      --output-format=csv
    outputs:
      highly_sensitive:
        cohort: output/measures/input_*.csv
  # Join static and time-varying variables
  join_cohorts_cvd:
    run: >
      cohort-joiner:v0.0.30
        --lhs output/measures/input_*.csv
        --rhs output/input_static.csv
        --output-dir output/measures/cvd
    needs: [generate_study_population, generate_study_population_static]
    outputs: 
      highly_sensitive:
        cohort: output/measures/cvd/input_*.csv
  join_cohorts_cvd_2020:
    run: >
      cohort-joiner:v0.0.30
        --lhs output/measures/input_*.csv
        --rhs output/input_static_2020.csv
        --output-dir output/measures/cvd/test
    needs: [generate_study_population, generate_study_population_static_2020]
    outputs: 
      highly_sensitive:
        cohort: output/measures/cvd/test/input_*.csv
  # Generates study population for mental health outcomes - start date = 2018-03-01
  generate_study_population_mh:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_mh --index-date-range "2018-03-01 to 2022-04-30 by month" --output-dir=output/measures --output-format=csv
    outputs:
      highly_sensitive:
        cohort: output/measures/input_mh_*.csv
  # Join static and time-varying variables
  join_cohorts_mh:
    run: >
      cohort-joiner:v0.0.30
        --lhs output/measures/input_mh_*.csv
        --rhs output/input_static.csv
        --output-dir output/measures/mh
    needs: [generate_study_population_mh, generate_study_population_static]
    outputs: 
      highly_sensitive:
        cohort: output/measures/mh/input_mh_*.csv

  # Check join
  #join_check:
   # run: stata-mp:latest analysis/variable_check.do 
    #needs: [join_cohorts_cvd, join_cohorts_mh, join_cohorts_cvd_2020]
    #outputs: 
    #  moderately_sensitive:
    #    log: logs/missing_check.log
  # Generates study population for diabetes outcomes
  generate_study_population_dm:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_dm --index-date-range "2018-03-01 to 2022-04-30 by month" --output-dir=output/measures/dm --output-format=csv 
    outputs:
      highly_sensitive:
        cohort: output/measures/dm/input_dm_*.csv

  # Generates study population for respiratory outcomes
  generate_study_population_resp:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_resp --index-date-range "2018-03-01 to 2022-04-30 by month" --output-dir=output/measures/resp --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/measures/resp/input_resp_*.csv

  # Generates study population for mortality
  generate_study_population_mortality:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_mortality --index-date-range "2019-02-01 to 2022-04-30 by month" --output-dir=output/measures/mortality --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/measures/mortality/input_mortality_*.csv
  
  # Generates study populations for baseline tables at 3 timepoints
  generate_study_population_tables_2019:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_tables --index-date-range "2019-01-01" --output-dir=output/measures/tables --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/measures/tables/input_tables_2019-01-01.csv
  generate_study_population_tables_2020:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_tables --index-date-range "2020-01-01" --output-dir=output/measures/tables --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/measures/tables/input_tables_2020-01-01.csv
  generate_study_population_tables_2021:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_tables --index-date-range "2021-01-01" --output-dir=output/measures/tables --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/measures/tables/input_tables_2021-01-01.csv
  
  # Generates baseline tables
  create_baseline_tables:
    run: stata-mp:latest analysis/101_baseline_tables.do
    needs: [generate_study_population_tables_2019, generate_study_population_tables_2020, generate_study_population_tables_2021]
    outputs:
      moderately_sensitive:
        log: logs/table1_descriptives.log
        output: output/tables/baseline_table_*.csv

  # Generates summary measures for CV 
  calculate_measures:
    run: cohortextractor:latest  generate_measures --study-definition study_definition --output-dir=output/measures/cvd
    needs: [join_cohorts_cvd]
    outputs:
      moderately_sensitive:
        measure: output/measures/cvd/measure_*_rate.csv
  # Generates summary measures for mental health outcomes
  calculate_measures_mh:
    run: cohortextractor:latest  generate_measures --study-definition study_definition_mh --output-dir=output/measures/mh
    needs: [join_cohorts_mh]
    outputs:
      moderately_sensitive:
        measure: output/measures/mh/measure_*_rate.csv
  # Generates summary measures for diabetes outcomes  
  calculate_measures_dm:
    run: cohortextractor:latest  generate_measures --study-definition study_definition_dm --output-dir=output/measures/dm
    needs: [generate_study_population_dm]
    outputs:
      moderately_sensitive:
        measure: output/measures/dm/measure_dm_*_rate.csv

  # Generates summary measures for respiratory outcomes  
  calculate_measures_resp:
    run: cohortextractor:latest  generate_measures --study-definition study_definition_resp --output-dir=output/measures/resp
    needs: [generate_study_population_resp]
    outputs:
      moderately_sensitive:
        measure: output/measures/resp/measure_*_rate.csv

  # Generates summary measures for mortality outcomes  
  calculate_measures_mortality:
    run: cohortextractor:latest  generate_measures --study-definition study_definition_mortality --output-dir=output/measures/mortality
    needs: [generate_study_population_mortality]
    outputs:
      moderately_sensitive:
        measure: output/measures/mortality/measure_*_rate.csv
  
  # Generates graphs for CVD & mental health outcomes
  graphs_mh_cvd:
    run: stata-mp:latest analysis/102_graphs_mh_cvd.do 
    needs: [calculate_measures, calculate_measures_mh]
    outputs: 
      moderately_sensitive:
        log: logs/graphs_mh_cvd.log
        output: output/graphs/line_*.svg
     
  # Generates graphs for diabetes outcomes
  graphs_dm:
    run: stata-mp:latest analysis/103_graphs_dm.do 
    needs: [calculate_measures_dm]
    outputs: 
      moderately_sensitive:
        log: logs/graphs_dm.log
        output: output/graphs/line_dm_*.svg
    
  # Generate graphs for respiratory outcomes
  graphs_resp:
    run: stata-mp:latest analysis/104_graphs_resp.do 
    needs: [calculate_measures_resp]
    outputs: 
      moderately_sensitive:
        log: logs/graphs_resp.log
        output: output/graphs/line_resp_*.svg

  # Generate graphs for mortality
  graphs_mortality:
    run: stata-mp:latest analysis/105_graphs_mortality.do 
    needs: [calculate_measures_mortality]
    outputs: 
      moderately_sensitive:
        log: logs/graphs_mortality.log
        output: output/graphs/line_mortality_*.svg
        
  # Running analysis scripts
  model_checks_dm:
    run: stata-mp:latest analysis/201_model_check_dm.do
    needs: [calculate_measures_dm]
    outputs:
      moderately_sensitive: 
        log: logs/model_checks_dm.log
        output: output/time_series/dm_*.svg

  model_checks_resp:
    run: stata-mp:latest analysis/202_model_check_resp.do
    needs: [calculate_measures_resp]
    outputs:
      moderately_sensitive: 
        log: logs/model_checks_resp.log
        output: output/time_series/resp_*.svg

  model_checks_cvd:
    run: stata-mp:latest analysis/203_model_check_cvd.do
    needs: [calculate_measures]
    outputs:
      moderately_sensitive: 
        log: logs/model_checks_cvd.log
        output: output/time_series/cvd_*.svg

  model_checks_mh:
    run: stata-mp:latest analysis/204_model_check_mh.do
    needs: [calculate_measures_mh]
    outputs:
      moderately_sensitive: 
        log: logs/model_checks_mh.log
        output: output/time_series/mh_*.svg

  model_checks_mortality:
    run: stata-mp:latest analysis/205_model_check_mortality.do
    needs: [calculate_measures_mortality]
    outputs:
      moderately_sensitive: 
        log: logs/model_checks_mortality.log
        output: output/time_series/mortality_*.svg
  
  time_series_resp:
    run: stata-mp:latest analysis/206_time_series_resp.do
    needs: [calculate_measures_resp]
    outputs:
      moderately_sensitive:
        log: logs/tsreg_resp.log
        output: output/time_series/tsreg_resp*.csv
        graph: output/time_series/margins_resp*.svg

  time_series_dm:
    run: stata-mp:latest analysis/207_time_series_dm.do
    needs: [calculate_measures_dm]
    outputs:
      moderately_sensitive:
        log: logs/tsreg_dm.log
        output: output/time_series/tsreg_dm*.csv
        graph: output/time_series/margins_dm*.svg

  time_series_cvd:
    run: stata-mp:latest analysis/208_time_series_cvd.do
    needs: [calculate_measures]
    outputs:
      moderately_sensitive:
        log: logs/tsreg_cvd.log
        output: output/time_series/tsreg_cvd_*.csv
        graphs: output/time_series/margins_cvd*.svg

  time_series_mh:
    run: stata-mp:latest analysis/209_time_series_mh.do
    needs: [calculate_measures_mh]
    outputs:
      moderately_sensitive:
        log: logs/tsreg_mh.log
        output: output/time_series/tsreg_mh*.csv
        graphs: output/time_series/margins_mh*.svg

  time_series_mortality:
    run: stata-mp:latest analysis/210_time_series_mortality.do
    needs: [calculate_measures_mortality]
    outputs:
      moderately_sensitive:
        log: logs/tsreg_mortality.log
        output: output/time_series/tsreg_mortality*.csv
        graph: output/time_series/margins_mort*.svg

  # Study definition for pre-pandemic survival analysis
  generate_study_population_survival_pre:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_survival_pre --output-dir=output/survival --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/survival/input_survival_pre*.csv

  # Study definition for pandemic survival analysis
  generate_study_population_survival_pandemic:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_survival_pandemic --output-dir=output/survival --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/survival/input_survival_pandemic*.csv

  # Study definition for pandemic waves
  # Using dates: wave 1 2020-03-23 to 2020-05-30, wave 2 2020-09-07 to 2021-04-24, wave 3 2021-05-28 to 2021-12-14
  # Easing period after each wave
  # dates match covid_mortality_over_time study - wave 1 & 2 based on ONS COVID infection survey, wave 3 based on >50% prevalence
  generate_study_population_survival_wave1:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_survival_wave1 --output-dir=output/survival --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/survival/input_survival_wave1*.csv

  # Study definition for pandemic survival analysis
  generate_study_population_survival_easing1:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_survival_easing1 --output-dir=output/survival --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/survival/input_survival_easing1*.csv

  # Study definition for pandemic survival analysis
  generate_study_population_survival_wave2:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_survival_wave2 --output-dir=output/survival --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/survival/input_survival_wave2*.csv

  # Study definition for pandemic survival analysis
  generate_study_population_survival_easing2:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_survival_easing2 --output-dir=output/survival --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/survival/input_survival_easing2*.csv

  # Study definition for pandemic survival analysis
  generate_study_population_survival_wave3:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_survival_wave3 --output-dir=output/survival --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/survival/input_survival_wave3*.csv

  # Study definition for pandemic survival analysis
  generate_study_population_survival_easing3:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_survival_easing3 --output-dir=output/survival --output-format=csv --skip-existing
    outputs:
      highly_sensitive:
        cohort: output/survival/input_survival_easing3*.csv

  # Cox models
  # Perpare datasets
  cox_model_prep:
    run: stata-mp:latest analysis/301_cox_model_prep.do
    needs: [generate_study_population_survival_pre, generate_study_population_survival_pandemic, generate_study_population_survival_wave1, generate_study_population_survival_easing1, generate_study_population_survival_wave2, generate_study_population_survival_easing2, generate_study_population_survival_wave3, generate_study_population_survival_easing3]
    outputs:
      highly_sensitive:
        output: output/prep_survival*.dta
      moderately_sensitive:
        log: logs/cox_model_prep.log

  # Check data
  cox_data_check:
    run: stata-mp:latest analysis/301_1_variable_check.do
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/variable_check.log
        
  # Repeat for each outcome
  cox_models_cvd:
    run: stata-mp:latest analysis/302_cox_models.do "cvd"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_cvd.log
        output: output/tables/cvd*.txt
        graph: output/graphs/km_cvd*.svg

  cox_models_mi:
    run: stata-mp:latest analysis/302_cox_models.do "mi"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_mi.log
        output: output/tables/mi*.txt
        graph: output/graphs/km_mi*.svg
  
  cox_models_stroke:
    run: stata-mp:latest analysis/302_cox_models.do "stroke"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_stroke.log
        output: output/tables/stroke*.txt
        graph: output/graphs/km_stroke*.svg

  cox_models_heart_failure:
    run: stata-mp:latest analysis/302_cox_models.do "hf"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_hf.log
        output: output/tables/hf*.txt
        graph: output/graphs/km_hf*.svg

  cox_models_vte:
    run: stata-mp:latest analysis/302_cox_models.do "vte"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_vte.log
        output: output/tables/vte*.txt
        graph: output/graphs/km_vte*.svg

  cox_models_t1dm:
    run: stata-mp:latest analysis/302_cox_models.do "t1dm"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_t1dm.log
        output: output/tables/t1dm*.txt
        graph: output/graphs/km_t1dm*.svg

  cox_models_t2dm:
    run: stata-mp:latest analysis/302_cox_models.do "t2dm"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_t2dm.log
        output: output/tables/t2dm*.txt
        graph: output/graphs/km_t2dm*.svg

  cox_models_dm_keto:
    run: stata-mp:latest analysis/302_cox_models.do "dm_keto"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_dm_keto.log
        output: output/tables/dm_keto*.txt
        graph: output/graphs/km_dm_keto*.svg

  cox_models_dm:
    run: stata-mp:latest analysis/302_cox_models.do "dm"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_dm.log
        output: output/tables/dm*.txt
        graph: output/graphs/km_dm*.svg

  cox_models_asthma:
    run: stata-mp:latest analysis/302_cox_models.do "asthma"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_asthma.log
        output: output/tables/asthma*.txt
        graph: output/graphs/km_asthma*.svg

  cox_models_copd:
    run: stata-mp:latest analysis/302_cox_models.do "copd"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_copd.log
        output: output/tables/copd*.txt
        graph: output/graphs/km_copd*.svg

  cox_models_depress:
    run: stata-mp:latest analysis/302_cox_models.do "depress"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_depress.log
        output: output/tables/depress*.txt
        graph: output/graphs/km_depress*.svg

  cox_models_anx:
    run: stata-mp:latest analysis/302_cox_models.do "anx"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_anx.log
        output: output/tables/anx*.txt
        graph: output/graphs/km_anx*.svg

  cox_models_smi:
    run: stata-mp:latest analysis/302_cox_models.do "smi"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_smi.log
        output: output/tables/smi*.txt
        graph: output/graphs/km_smi*.svg

  cox_models_mh:
    run: stata-mp:latest analysis/302_cox_models.do "mh"
    needs: [cox_model_prep]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_mh.log
        output: output/tables/mh*.txt
        graph: output/graphs/km_mh*.svg

  cox_models_graphs:
    run: stata-mp:latest analysis/303_cox_models_graphs.do
    needs: [generate_study_population_survival_pre, generate_study_population_survival_pandemic]
    outputs:
      moderately_sensitive:
        log: logs/cox_model_graphs.log
        graph: output/survival/estimates_*.svg
      
  cox_outputs:
    run: stata-mp:latest analysis/304_prep_r_vis.do
    needs: [cox_models_anx, cox_models_depress, cox_models_asthma, cox_models_copd, cox_models_dm_keto, cox_models_t1dm, cox_models_t2dm, cox_models_heart_failure, cox_models_mi, cox_models_stroke, cox_models_vte]
    outputs:
      moderately_sensitive:
        log: logs/prep_r_vis.log
        output: output/tables/all_cox_data*.csv