project.yaml

version: '3.0'

expectations:
  population_size: 100000

actions:

# Extract data
# When argument --index-date-range is changed, change has to be made in ./analysis/config.json too
  generate_study_population:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition 
        --skip-existing 
        --output-format=csv.gz 
        --index-date-range "2020-03-01 to 2022-02-01 by month"
    outputs:
      highly_sensitive:
        cohort: output/input_*.csv.gz

# Extract ethnicity
  generate_study_population_ethnicity:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_ethnicity 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_ethnicity.csv.gz

# Join data
  join_cohorts:
    run: >
      cohort-joiner:v0.0.7
        --lhs output/input_202*.csv.gz
        --rhs output/input_ethnicity.csv.gz
        --output-dir=output/joined
    needs: [generate_study_population, generate_study_population_ethnicity]
    outputs:
      highly_sensitive:
        cohort: output/joined/input_202*.csv.gz

# Calculate mortality rates (crude + subgroup specific)
  calculate_measures:
    run: >
      cohortextractor:latest generate_measures 
        --study-definition study_definition
        --skip-existing
        --output-dir=output/joined
    needs: [join_cohorts]
    outputs:
      moderately_sensitive:
        measure: output/joined/measure_*_mortality_rate.csv

# Calculate mortality rates ckd_rrt subgroup
  calculate_measures_ckd_rrt:
    run: r:latest analysis/measures_calc_ckd_rrt.R
    needs: [join_cohorts]
    outputs:
      moderately_sensitive:
        measure: output/joined/measure_ckd_rrt_mortality_rate.csv

# Redact rates
  redact_rates:
    run: r:latest analysis/utils/redact_rates.R
    needs: [calculate_measures, calculate_measures_ckd_rrt]
    outputs:
      moderately_sensitive:
        csvs: output/rates/redacted/*_redacted.csv       

# Standardise crude mortality rate
  standardise_crude_rates:
    run: r:latest analysis/crude_rates_standardise.R
    needs: [redact_rates]
    outputs:
      moderately_sensitive:
        csvs: output/rates/standardised/crude_*std.csv 

# Standardise subgroup specific mortality rates
  standardise_subgroup_rates:
    run: r:latest analysis/subgroups_rates_standardise.R
    needs: [redact_rates]
    outputs:
      moderately_sensitive:
        csvs: output/rates/standardised/*_std.csv

# Process subgroup specific mortality rates
  process_subgroup_rates:
    run: r:latest analysis/utils/process_rates.R
    needs: [standardise_subgroup_rates, standardise_subgroup_rates]
    outputs:
      moderately_sensitive:
        csvs: output/rates/processed/*.csv

# Calculate standardised rate ratios
  calculate_rate_ratios:
    run: r:latest analysis/subgroups_ratios.R
    needs: [standardise_subgroup_rates, process_subgroup_rates]
    outputs:
      moderately_sensitive:
        csvs: output/ratios/*.csv

# Plot and save graphs depicting the crude rates
  visualise_crude_rates:
    run: r:latest analysis/crude_rates_visualise.R
    needs: [standardise_crude_rates]
    outputs:
      moderately_sensitive:
        pngs: output/figures/rates_crude/*.png

# Plot and save graphs depicting the subgroup specific mortality rates
  visualise_subgroup_rates:
    run: r:latest analysis/subgroups_rates_visualise.R
    needs: [standardise_subgroup_rates, process_subgroup_rates]
    outputs:
      moderately_sensitive:
        pngs: output/figures/rates_subgroups/*.png

# Plot and save graphs depicting the subgroup specific mortality ratios
  visualise_subgroup_ratios:
    run: r:latest analysis/subgroups_ratios_visualise.R
    needs: [calculate_rate_ratios]
    outputs:
      moderately_sensitive:
        pngs: output/figures/ratios_subgroups/*.png

# SECOND PART OF STUDY
  generate_study_population_flowchart:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_flowchart
        --skip-existing 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_flowchart.csv.gz
  
  generate_study_population_flowchart2:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_flowchart2
        --skip-existing 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_flowchart2.csv.gz

  generate_study_population_flowchart3:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_flowchart3
        --skip-existing 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_flowchart3.csv.gz

  generate_study_population_flowchart4:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_flowchart4
        --skip-existing 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_flowchart4.csv.gz
        
  generate_study_population_flowchart5:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_flowchart5
        --skip-existing 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_flowchart5.csv.gz

  # Process data flowchart
  process_data_flowchart:
    run: r:latest analysis/data_flowchart_process.R
    needs: [generate_study_population_flowchart, generate_study_population_flowchart2, generate_study_population_flowchart3, generate_study_population_flowchart4, generate_study_population_flowchart5]
    outputs:
      highly_sensitive: 
        rds1: output/processed/input_flowchart_wave1.rds
        rds2: output/processed/input_flowchart_wave2.rds
        rds3: output/processed/input_flowchart_wave3.rds
        rds4: output/processed/input_flowchart_wave4.rds
        rds5: output/processed/input_flowchart_wave5.rds
  
  # Skim data flowchart
  skim_data_flowchart:
    run: r:latest analysis/data_skim.R output/processed/input_flowchart_wave1.rds output/data_properties
    needs: [process_data_flowchart]
    outputs: 
      moderately_sensitive:
        txt1: output/data_properties/input_flowchart_wave1_skim.txt
        txt2: output/data_properties/input_flowchart_wave1_coltypes.txt
        txt3: output/data_properties/input_flowchart_wave1_tabulate.txt

  # Numbers for flowchart
  calc_numbers_flowchart_wave1:
    run: r:latest analysis/flowchart.R wave1 output/tables/flowchart
    needs: [process_data_flowchart]
    outputs:
      moderately_sensitive:
        cohort: output/tables/flowchart/wave1_flowchart.csv

  # Numbers for flowchart
  calc_numbers_flowchart_wave2:
    run: r:latest analysis/flowchart.R wave2 output/tables/flowchart
    needs: [process_data_flowchart]
    outputs:
      moderately_sensitive:
        cohort: output/tables/flowchart/wave2_flowchart.csv

  # Numbers for flowchart
  calc_numbers_flowchart_wave3:
    run: r:latest analysis/flowchart.R wave3 output/tables/flowchart
    needs: [process_data_flowchart]
    outputs:
      moderately_sensitive:
        cohort: output/tables/flowchart/wave3_flowchart.csv

  # Numbers for flowchart
  calc_numbers_flowchart_wave4:
    run: r:latest analysis/flowchart.R wave4 output/tables/flowchart
    needs: [process_data_flowchart]
    outputs:
      moderately_sensitive:
        cohort: output/tables/flowchart/wave4_flowchart.csv
        
  # Numbers for flowchart
  calc_numbers_flowchart_wave5:
    run: r:latest analysis/flowchart.R wave5 output/tables/flowchart
    needs: [process_data_flowchart]
    outputs:
      moderately_sensitive:
        cohort: output/tables/flowchart/wave5_flowchart.csv

  generate_study_population_wave1:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_wave1 
        --skip-existing 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_wave1.csv.gz

  generate_study_population_wave2:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_wave2
        --skip-existing 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_wave2.csv.gz

  generate_study_population_wave3:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_wave3
        --skip-existing 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_wave3.csv.gz
  
  generate_study_population_wave4:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_wave4
        --skip-existing 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_wave4.csv.gz

  generate_study_population_wave5:
    run: >
      cohortextractor:latest generate_cohort 
        --study-definition study_definition_wave5
        --skip-existing 
        --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_wave5.csv.gz

# Join data
  join_cohorts_waves:
    run: >
      cohort-joiner:v0.0.7
        --lhs output/input_wave*.csv.gz
        --rhs output/input_ethnicity.csv.gz
        --output-dir=output/joined
    needs: [generate_study_population_wave1, generate_study_population_wave2, generate_study_population_wave3, generate_study_population_wave4, generate_study_population_wave5, generate_study_population_ethnicity]
    outputs:
      highly_sensitive:
        cohort: output/joined/input_wave*.csv.gz

# Process data
  process_data_wave1:
    run: r:latest analysis/data_process.R wave1
    needs: [join_cohorts_waves]
    outputs:
      highly_sensitive: 
        rds: output/processed/input_wave1.rds

  process_data_wave2:
    run: r:latest analysis/data_process.R wave2
    needs: [join_cohorts_waves]
    outputs:
      highly_sensitive: 
        rds: output/processed/input_wave2.rds

  process_data_wave3:
    run: r:latest analysis/data_process.R wave3
    needs: [join_cohorts_waves]
    outputs:
      highly_sensitive: 
        rds: output/processed/input_wave3.rds

  process_data_wave4:
    run: r:latest analysis/data_process.R wave4
    needs: [join_cohorts_waves]
    outputs:
      highly_sensitive: 
        rds: output/processed/input_wave4.rds

  process_data_wave5:
    run: r:latest analysis/data_process.R wave5
    needs: [join_cohorts_waves]
    outputs:
      highly_sensitive: 
        rds: output/processed/input_wave5.rds

# Skim data
  skim_data_wave1:
    run: r:latest analysis/data_skim.R output/processed/input_wave1.rds output/data_properties
    needs: [process_data_wave1]
    outputs: 
      moderately_sensitive:
        txt1: output/data_properties/input_wave1_skim.txt
        txt2: output/data_properties/input_wave1_coltypes.txt
        txt3: output/data_properties/input_wave1_tabulate.txt

  skim_data_wave2:
    run: r:latest analysis/data_skim.R output/processed/input_wave2.rds output/data_properties
    needs: [process_data_wave2]
    outputs: 
      moderately_sensitive:
        txt1: output/data_properties/input_wave2_skim.txt
        txt2: output/data_properties/input_wave2_coltypes.txt
        txt3: output/data_properties/input_wave2_tabulate.txt

  skim_data_wave3:
    run: r:latest analysis/data_skim.R output/processed/input_wave3.rds output/data_properties
    needs: [process_data_wave3]
    outputs: 
      moderately_sensitive:
        txt1: output/data_properties/input_wave3_skim.txt
        txt2: output/data_properties/input_wave3_coltypes.txt
        txt3: output/data_properties/input_wave3_tabulate.txt

  skim_data_wave4:
    run: r:latest analysis/data_skim.R output/processed/input_wave4.rds output/data_properties
    needs: [process_data_wave4]
    outputs: 
      moderately_sensitive:
        txt1: output/data_properties/input_wave4_skim.txt
        txt2: output/data_properties/input_wave4_coltypes.txt
        txt3: output/data_properties/input_wave4_tabulate.txt
  
  skim_data_wave5:
    run: r:latest analysis/data_skim.R output/processed/input_wave5.rds output/data_properties
    needs: [process_data_wave5]
    outputs: 
      moderately_sensitive:
        txt1: output/data_properties/input_wave5_skim.txt
        txt2: output/data_properties/input_wave5_coltypes.txt
        txt3: output/data_properties/input_wave5_tabulate.txt

# Missings in bmi, ethnicity and smoking
  calc_missings:
    run: r:latest analysis/calc_missings.R
    needs: [process_data_wave1, process_data_wave2, process_data_wave3, process_data_wave4, process_data_wave5]
    outputs: 
      moderately_sensitive:
        csvs: output/tables/missings/waves_missings.csv

# Incidence rates (crude)
  calc_irs:
    run: r:latest analysis/waves_irs.R
    needs: [process_data_wave1, process_data_wave2, process_data_wave3, process_data_wave4, process_data_wave5]
    outputs: 
      moderately_sensitive:
        csvs: output/tables/wave*_ir.csv
        csv: output/tables/ir_crude.csv

# Create table one
  create_table_one:
    run: r:latest analysis/table_one.R
    needs: [process_data_wave1, process_data_wave2, process_data_wave3, process_data_wave4, process_data_wave5, calc_irs]
    outputs: 
      moderately_sensitive:
        csv: output/tables/table1.csv

# Incidence rates (crude)
  calc_irs_std:
    run: r:latest analysis/waves_std_irs.R
    needs: [process_data_wave1, process_data_wave2, process_data_wave3, process_data_wave4, process_data_wave5]
    outputs: 
      moderately_sensitive:
        csvs: output/tables/wave*_ir_std.csv

# Create table A1 (table with IRs)
  create_table_A1:
    run: r:latest analysis/table_A1.R
    needs: [calc_irs, calc_irs_std]
    outputs: 
      moderately_sensitive:
        csv: output/tables/table_A1.csv

# Kaplan-Meier
  create_kaplan_meier:
    run: r:latest analysis/waves_kaplan_meier.R
    needs: [process_data_wave1, process_data_wave2, process_data_wave3, process_data_wave4, process_data_wave5]
    outputs:
      moderately_sensitive:
        pngs: output/figures/kaplan_meier/wave*_*.png

# COX ph models
  model_cox_ph_wave1:
    run: r:latest analysis/waves_model_survival.R wave1 output/tables
    needs: [process_data_wave1]
    outputs: 
      moderately_sensitive:
        csvs1: output/tables/wave1_effect_estimates.csv
        csvs2: output/tables/wave1_ph_tests.csv
        csvs3: output/tables/wave1_log_file.csv

  model_cox_ph_wave2:
    run: r:latest analysis/waves_model_survival.R wave2 output/tables
    needs: [process_data_wave2]
    outputs: 
      moderately_sensitive:
        csvs1: output/tables/wave2_effect_estimates.csv
        csvs2: output/tables/wave2_ph_tests.csv
        csvs3: output/tables/wave2_log_file.csv

  model_cox_ph_wave3:
    run: r:latest analysis/waves_model_survival.R wave3 output/tables
    needs: [process_data_wave3]
    outputs: 
      moderately_sensitive:
        csvs1: output/tables/wave3_effect_estimates.csv
        csvs2: output/tables/wave3_ph_tests.csv
        csvs3: output/tables/wave3_log_file.csv

  model_cox_ph_wave4:
    run: r:latest analysis/waves_model_survival.R wave4 output/tables
    needs: [process_data_wave4]
    outputs: 
      moderately_sensitive:
        csvs1: output/tables/wave4_effect_estimates.csv
        csvs2: output/tables/wave4_ph_tests.csv
        csvs3: output/tables/wave4_log_file.csv

  model_cox_ph_wave5:
    run: r:latest analysis/waves_model_survival.R wave5 output/tables
    needs: [process_data_wave5]
    outputs: 
      moderately_sensitive:
        csvs1: output/tables/wave5_effect_estimates.csv
        csvs2: output/tables/wave5_ph_tests.csv
        csvs3: output/tables/wave5_log_file.csv

# Create table two
  create_table_two:
    run: r:latest analysis/table_two.R
    needs: [model_cox_ph_wave1, model_cox_ph_wave2, model_cox_ph_wave3, model_cox_ph_wave4, model_cox_ph_wave5]
    outputs:
      moderately_sensitive:
        csv: output/tables/table2.csv

# Tidy absrisks (IRs) for viz
  tidy_absrisks_for_viz:
    run: r:latest analysis/absrisks_tidy_for_viz.R
    needs: [calc_irs, calc_irs_std]
    outputs:
      moderately_sensitive: 
        csv: output/tables/absrisks_for_viz_tidied.csv

# Tidy relrisks (HRs) for viz
  tidy_relrisks_for_viz:
    run: r:latest analysis/relrisks_tidy_for_viz.R
    needs: [model_cox_ph_wave1, model_cox_ph_wave2, model_cox_ph_wave3, model_cox_ph_wave4, model_cox_ph_wave5]
    outputs:
      moderately_sensitive: 
        csv: output/tables/relrisks_for_viz_tidied.csv

# Vaccine data
  vax_analysis:
    run: r:latest analysis/waves_vax_coverage.R
    needs: [process_data_wave1, process_data_wave2, process_data_wave3, process_data_wave4, process_data_wave5]
    outputs:
      moderately_sensitive: 
        csv1: output/tables/vax/wave*_vax_*.csv

# Pos test x covid mort
  describe_pos_tests:
    run: r:latest analysis/waves_pos_tests.R
    needs: [process_data_wave1, process_data_wave2, process_data_wave3, process_data_wave4, process_data_wave5]
    outputs:
      moderately_sensitive: 
        csv: output/tables/pos_test_in_covid_deaths.csv

# Wave-specific HR plots
#  plot_relrisks:
#    run: r:latest -e 'rmarkdown::render("analysis/wave_HRs.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output/markdown")'
#    needs: [tidy_relrisks_for_viz]
#    outputs:
#      moderately_sensitive:
#        csv: output/markdown/wave_HRs.html