project.yaml

version: "3.0"

expectations:
  population_size: 1000

actions:

  generate_study_population_calculators:
    run: cohortextractor:latest generate_cohort
      --study-definition study_definition_calculators
      --output-format csv.gz
      --with-end-date-fix
    outputs:
      highly_sensitive:
        cohort: output/input_calculators.csv.gz


  calculate_calculators:
    run: python:latest python analysis/scripts/calculate_calculators.py 
    needs: [generate_study_population_calculators]
    outputs:
      highly_sensitive:
        counts: output/input_calculators_calculated.csv.gz

  crcl_gault:
    run: python:latest python analysis/scripts/crcl_gault.py
    needs: [calculate_calculators]
    outputs:
      moderately_sensitive:
        plots: output/dist*.png
        venn: output/venn*.png
        counts: output/creatinine_vs_crlc.json

  generate_study_population:
    run: cohortextractor:latest generate_cohort
      --study-definition study_definition
      --index-date-range "2019-01-01 to 2023-07-01 by month"
      --output-format csv.gz
      --with-end-date-fix
    outputs:
      highly_sensitive:
        cohort: output/input*.csv.gz

  generate_study_population_ukrr_ethnicity:
    run: cohortextractor:latest generate_cohort 
      --study-definition study_definition_ukrr_ethnicity
      --output-dir=output
      --output-format csv.gz
      --with-end-date-fix
    outputs:
      highly_sensitive:
        cohort: output/input_ukrr_ethnicity.csv.gz

  join_cohorts:
    run: >
      cohort-joiner:v0.0.46
        --lhs output/input_20*.csv.gz
        --rhs output/input_ukrr_ethnicity.csv.gz
        --output-dir output/joined
    needs: [generate_study_population, generate_study_population_ukrr_ethnicity]
    outputs:
      highly_sensitive:
        cohort: output/joined/input_20*.csv.gz

  generate_table_1:
    run: python:latest python analysis/scripts/table_1.py --study_def_paths="output/joined/input_*.csv.gz" --demographics="age_band,sex,region,imd,ethnicity"
    needs: [join_cohorts]
    outputs:
      moderately_sensitive:
        tables: output/pub/descriptive_tables/table_1*.csv

  get_counts:
    run: python:latest python analysis/scripts/combine_operators.py
    needs: [join_cohorts]
    outputs:
      moderately_sensitive:
        counts_pub: output/pub/operator_counts/*.csv
  

  generate_top_5_tables:
    run: python:latest python analysis/scripts/top_codes.py
    needs: [get_counts]
    outputs:
      moderately_sensitive:
        counts: output/pub/top_5_tables/top_5_code_table_*.csv

  ckd_discrepancies:
    run: python:latest python analysis/scripts/ckd_discrepancies.py
    needs: [join_cohorts]
    outputs:
      moderately_sensitive:
        tables: output/pub/ckd_overlap/ckd_staging_*.csv

  ukrr_vs_prim_care_ckd:
    run: python:latest python analysis/scripts/ukrr_prim_care_crossover_ckd.py
    needs: [join_cohorts]
    outputs:
      moderately_sensitive:
        upset: output/pub/ukrr_pc_overlap/*.jpeg
        table: output/pub/ukrr_pc_overlap/*.csv

  generate_measures:
    run: cohortextractor:latest generate_measures 
      --study-definition study_definition
      --output-dir=output/joined
    needs: [join_cohorts]
    outputs:
      moderately_sensitive:
        measure_csv: output/joined/measure_*_rate.csv

  
  generate_plots:
    run: python:latest python analysis/scripts/plot_measures.py
    needs: [generate_measures, join_cohorts]
    outputs:
      moderately_sensitive:
        counts: output/pub/deciles/figures/plot_*.jpeg
        csv: output/pub/deciles/data/plot_*.csv
        ckd_stage: output/pub/ckd_stage/plot_ckd_stage.jpeg
        ckd_stage_csv: output/pub/ckd_stage/plot_ckd_stage.csv
        tests_ckd_stage: output/pub/tests_by_ckd_stage/*.jpeg
        ukrr: output/pub/ukrr_testing/*.jpeg
        ukrr_data: output/pub/ukrr_testing/*.csv

  generate_plots_numeric_values:
    run: python:latest python analysis/scripts/plot_numeric_values.py
    needs: [join_cohorts]
    outputs:
      moderately_sensitive:
        figures: output/pub/numeric_values/*dis*.jpeg
        tables: output/pub/numeric_values/*dis*.csv

  generate_study_population_ukrr_jul_22:
    run: cohortextractor:latest generate_cohort
      --study-definition study_definition_ukrr
      --index-date-range "2022-07-01"
      --output-format csv.gz
      --with-end-date-fix
    outputs:
      highly_sensitive:
        cohort: output/ukrr/input_ukrr_2022-07-01.csv.gz

  generate_study_population_ukrr:
    run: cohortextractor:latest generate_cohort
      --study-definition study_definition_ukrr
      --index-date-range "2020-01-01"
      --output-format csv.gz
      --with-end-date-fix
    outputs:
      highly_sensitive:
        cohort: output/ukrr/input_ukrr*.csv.gz

  generate_study_population_ukrr_1:
    run: cohortextractor:latest generate_cohort
      --study-definition study_definition_ukrr_short
      --index-date-range "2021-01-01"
      --output-format csv.gz
      --with-end-date-fix
    outputs:
      highly_sensitive:
        cohort: output/ukrr/input_ukr*.csv.gz

  generate_study_population_ukrr_2:
    run: cohortextractor:latest generate_cohort
      --study-definition study_definition_ukrr_short
      --index-date-range "2022-01-01"
      --output-format csv.gz
      --with-end-date-fix
    outputs:
      highly_sensitive:
        cohort: output/ukrr/input_uk*.csv.gz

  generate_study_population_paeds:
    run: cohortextractor:latest generate_cohort
      --study-definition study_definition_paeds
      --output-format csv.gz
      --with-end-date-fix
    outputs:
      highly_sensitive:
        cohort: output/input_paeds.csv.gz


  

  generate_study_population_check_SC:
    run: cohortextractor:latest generate_cohort
      --study-definition study_definition_check_SC
      --output-format csv.gz
      --with-end-date-fix
    outputs:
      highly_sensitive:
        cohort: output/input_check_SC.csv.gz

  

  # generate_notebook:
  #   run: jupyter:latest jupyter nbconvert /workspace/analysis/report.ipynb --execute --to html --template basic --output-dir=/workspace/output --ExecutePreprocessor.timeout=86400 --no-input
  #   needs: [generate_plots, get_counts]
  #   outputs:
  #     moderately_sensitive:
  #       notebook: output/report.html

  # generate_ukrr_checks:
  #  run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/ukrr-checks.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #  needs: [generate_study_population, generate_study_population_ukrr_ethnicity]
  #  outputs:
  #    moderately_sensitive:
  #      report: output/notebooks/ukrr-checks.html

  #generate_ukrr_report:
  #  run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/ukrr-report.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #  needs: [join_acr]
  #  outputs:
  #    moderately_sensitive:
  #      report: output/notebooks/ukrr-report.html
       

  # generate_ukrr_discrepancies:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/ukrr-discrepancies.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [join_acr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/ukrr-discrepancies.html



  
  # generate_kidney_week:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/kidney_week.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [join_acr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/kidney_week.html
  
  # generate_ukrr_tables_figures:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/ukrr-tables-figures.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/ukrr-tables-figures.html

  generate_ukrr_tables_figures_1:
    run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/ukrr-tables-figures-2021.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
    needs: [generate_study_population_ukrr_1]
    outputs:
      moderately_sensitive:
        report: output/notebooks/ukrr-tables-figures-2021.html

  generate_ukrr_tables_figures_2:
    run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/ukrr-tables-figures-2022.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
    needs: [generate_study_population_ukrr_2]
    outputs:
      moderately_sensitive:
        report: output/notebooks/ukrr-tables-figures-2022.html

  generate_false_positives_ckd:
    run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/false-positives-ckd.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
    needs: [generate_study_population_ukrr]
    outputs:
      moderately_sensitive:
        report: output/notebooks/false-positives-ckd.html

  generate_agreement_by_eGFR:
    run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/agreement-by-eGFR.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
    needs: [generate_study_population_ukrr]
    outputs:
      moderately_sensitive:
        report: output/notebooks/agreement-by-eGFR.html

  # generate_pc_sc_codes1a:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/pc-sc-codes1a.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/pc-sc-codes1a.html
  
  # generate_pc_sc_codes1b:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/pc-sc-codes1b.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/pc-sc-codes1b.html

  # generate_pc_sc_codes2:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/pc-sc-codes2.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/pc-sc-codes2.html

  # generate_pc_sc_codes3:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/pc-sc-codes3.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/pc-sc-codes3.html

  # generate_pc_sc_codes4a:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/pc-sc-codes4a.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/pc-sc-codes4a.html

  # generate_pc_sc_codes4b:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/pc-sc-codes4b.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/pc-sc-codes4b.html

  # generate_pc_sc_codes4c:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/pc-sc-codes4c.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/pc-sc-codes4c.html

  # generate_pc_sc_codes5a:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/pc-sc-codes5a.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/pc-sc-codes5a.html
  
  # generate_pc_sc_codes5b:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/pc-sc-codes5b.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/pc-sc-codes5b.html

  # generate_ukrr_tables_figures_paeds:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/ukrr-tables-figures-paeds.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_paeds]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/ukrr-tables-figures-paeds.html
  
  # generate_check_SC:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/check-SC.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_check_SC, generate_study_population_ukrr_ethnicity]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/check-SC.html

  # generate_modality_difference_table:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/modality-difference-table.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/modality-difference-table.html

  # generate_incident_analysis:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/incident-analysis.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/incident-analysis.html
        
  # generate_study_population_flowchart:
  #   run: cohortextractor:latest generate_cohort
  #     --study-definition study_definition_flowchart
  #     --output-format csv.gz
  #   outputs:
  #     highly_sensitive:
  #       cohort: output/input_flowchart.csv.gz
  
  # generate_median_age:
  #   run: r:latest -e 'rmarkdown::render("analysis/ukrr-analysis/median_age.Rmd", knit_root_dir = "/workspace", output_dir = "output/notebooks")'
  #   needs: [generate_study_population_ukrr]
  #   outputs:
  #     moderately_sensitive:
  #       report: output/notebooks/median_age.html