Parallel pathway file generation

Makefile

@@ -4,6 +4,10 @@
 # singularity exec singularity/depmap.sif RScript
 RSCRIPT_CMD ?= Rscript
 
+# Defines the number intermediate pathway data files that will be created. This allow generating the pathway data in parallel.
+# Changing this number requires an update to the data/master_positive.RData and data/master_negative.RData rules below.
+NUM_SUBSET_FILES ?= 10
+
 # The first target is the default, it makes "all" the data. Does not include container_image
 all: gene_summary depmap_data depmap_stats depmap_tables depmap_pathways
 
@@ -46,6 +50,11 @@ data/master_top_table.RData data/master_bottom_table.RData: code/generate_depmap
 	@echo "Creating depmap tables"
 	$(RSCRIPT_CMD) code/generate_depmap_tables.R
 
-data/master_positive.RData data/master_negative.RData: code/generate_depmap_pathways.R data/gene_summary.RData data/gene_summary.RData data/19Q4_achilles_cor.RData data/achilles_lower.Rds data/achilles_upper.Rds
-	@echo "Creating depmap pathways"
-	$(RSCRIPT_CMD) code/generate_depmap_pathways.R
+data/master_positive.RData: code/generate_pathways.sh code/generate_depmap_pathways.R code/merge_depmap_pathways.R
+	@echo "Creating positive pathways data"
+	RSCRIPT_CMD=$(RSCRIPT_CMD) PATHWAY_TYPE=positive NUM_SUBSET_FILES=$(NUM_SUBSET_FILES) ./code/generate_pathways.sh
+
+data/master_negative.RData: code/generate_pathways.sh code/generate_depmap_pathways.R code/merge_depmap_pathways.R
+	@echo "Creating negative pathways data"
+	RSCRIPT_CMD=$(RSCRIPT_CMD) PATHWAY_TYPE=negative NUM_SUBSET_FILES=$(NUM_SUBSET_FILES) ./code/generate_pathways.sh
+

README.md

@@ -46,6 +46,4 @@ To generate the data files, run:
 2. code/generate_depmap_data.R
 3. code/generate_depmap_stats.R
 4. code/generate_depmap_tables.R
-5. code/generate_depmap_pathways.R
-
-The files generated in steps 1-3 are required for steps 4 and 5. Step 4 takes about 60' to run locally. Step 5 requires some parallization, and you'll see objects dec(ile)1-10 that could be run in parallel. The code for step 5 has `gene_group <- sample` so it can be tested.
+5. code/generate_pathways.sh

build-slurm.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 #SBATCH --mem=32G
+#SBATCH --cpus-per-task=10
 #SBATCH --output=logs/ddh-%j.out
 
 source config.sh

code/depmap_pathways_util.R

@@ -0,0 +1,49 @@
+# Utility functions for use with depmap pathway generation
+library(here)
+
+dpu_pathways_positive_type <- "positive"
+dpu_pathways_negative_type <- "negative"
+
+# Get a path to a subset file within the data directory for either positive or negative pathway data
+dpu_get_subset_filepath <- function(pathways_type, file_idx, num_subset_files) {
+  # create filename like 'positive_subset_1_of_10' or 'negative_subset_1_of_10'
+  subset_filename <- paste0(pathways_type, "_subset", "_", file_idx, "_of_", num_subset_files, ".Rds")
+  here::here("data", subset_filename)
+}
+
+# Get a path all subset files within the data directory for either positive or negative pathway data
+dpu_get_all_pathways_subset_filepaths <- function(pathways_type, num_subset_files) {
+  dpu_get_subset_filepath(pathways_type, seq(num_subset_files), num_subset_files)
+}
+
+# parse command line for --type, --num-subset-files and optionally --idx
+dpu_parse_command_line <- function (include_idx) {
+  option_list <- list(
+    make_option(c("--type"), type="character", default="positive", dest='pathways_type',
+                help="Type of pathways file to create either 'positive' or 'negative'"),
+    make_option(c("--num-subset-files"), type="integer", dest='num_subset_files',
+                help="Number of subset pathways files")
+  )
+  if (include_idx) {
+    option_list <- c(
+      option_list,
+      make_option(c("--idx"), type="integer", dest='subset_file_idx',
+                  help="Specifies single pathways subset file to create.")
+    )
+  }
+  opt_parser <- OptionParser(option_list=option_list)
+  opt <- parse_args(opt_parser)
+  if (opt$pathways_type != dpu_pathways_positive_type && opt$pathways_type != dpu_pathways_negative_type) {
+    message("The --type argument must be either '", dpu_pathways_positive_type, "' or '", dpu_pathways_positive_type, "'")
+    q(status=1)
+  }
+  if (is.null(opt$num_subset_files)) {
+    message("The --num-subset-files flag is required.")
+    q(status=1)
+  }
+  if (include_idx && is.null(opt$subset_file_idx)) {
+    message("The --idx is required.")
+    q(status=1)
+  }
+  opt
+}