Deduplication of numerous files

coderbuild/beatAML/GetBeatAML.py

@@ -124,6 +124,7 @@ def generate_samples_file(prev_samples_path):
     mapping = {labId: i for i, labId in enumerate(all_samples['other_id'].unique(), start=(int(maxval)+1))}
     all_samples['improve_sample_id'] = all_samples['other_id'].map(mapping)
     all_samples.insert(1, 'improve_sample_id', all_samples.pop('improve_sample_id'))
+    all_samples.drop_duplicates(inplace=True)
     all_samples.to_csv("/tmp/beataml_samples.csv", index=False)
     return all_samples
 

coderbuild/broad_sanger/02-broadSangerOmics.R

@@ -668,9 +668,9 @@ main<-function(){
 
     lapply(alltypes,function(dt){
         print(dt)
-        temps<-sanger_files(sanger_filenames[[dt]],dt)|>tidyr::drop_na()
+        temps<-sanger_files(sanger_filenames[[dt]],dt)|>tidyr::drop_na()|>dplyr::distinct()
         readr::write_csv(temps,file=paste0('/tmp/sanger_',dt,'.csv.gz'))
-        tempd<-depmap_files(depmap_filenames[[dt]],dt)|>tidyr::drop_na()
+        tempd<-depmap_files(depmap_filenames[[dt]],dt)|>tidyr::drop_na()|>dplyr::distinct()
         readr::write_csv(tempd,file=paste0('/tmp/broad_',dt,'.csv.gz'))
 
 #        readr::write_csv(rbind(tempd,temps),file=paste0('/tmp/broad_sanger_',dt,'.csv.gz'))

coderbuild/broad_sanger/04-drug_dosage_and_curves.py

@@ -55,7 +55,7 @@
 for of in outfiles:
     final_file.append(pd.read_csv(of,sep='\t'))
 
-pd.concat(final_file).to_csv('/tmp/broad_sanger_experiments.tsv',index=False,sep='\t')
+pd.concat(final_file).drop_duplicates().to_csv('/tmp/broad_sanger_experiments.tsv',index=False,sep='\t')
 #os.system('cat *.0 > /tmp/broad_sanger_experiments.tsv')
 #os.system('gzip -f /tmp/experiments.tsv')
 

coderbuild/colorectal/02-omics-colorectal.py

@@ -242,6 +242,7 @@ def map_copy_number(copy_number_data, improve_id_data, entrez_data):
         else:
             print("Starting transcriptomics data.")
             transcriptomics_df = map_transcriptomics(transciptomics_data = "/tmp/GSE65253_col_tum_org_merge.csv.gz", improve_id_data = "/tmp/colorectal_samples.csv", entrez_data = "/tmp/genes.csv")
+            transcriptomics_df.drop_duplicates(inplace=True)
             transcriptomics_df.to_csv("/tmp/colorectal_transcriptomics.csv", index=False)
 
     if args.mutations:

coderbuild/liver/02-omics-liver.py

@@ -362,6 +362,7 @@ def map_proteomics(proteomics_data, improve_id_data, entrez_data):
         else:
             print("Starting transcriptomics data.")
             transcriptomics_df = map_transcriptomics(transciptomics_data = "/tmp/raw_rnaseq_data.csv", improve_id_data = "/tmp/liver_samples.csv", entrez_data = "/tmp/genes.csv")
+            transcriptomics_df.drop_duplicates(inplace=True)
             transcriptomics_df.to_csv("/tmp/liver_transcriptomics.csv", index=False)
 
     if args.mutations:
@@ -385,8 +386,9 @@ def map_proteomics(proteomics_data, improve_id_data, entrez_data):
             exit()
         else:
             print("Starting copy number data.")
-            mutation_df = map_copy_number(copy_number_data = "/tmp/raw_copynum_data.csv", improve_id_data = "/tmp/liver_samples.csv", entrez_data = "/tmp/genes.csv")
-            mutation_df.to_csv("/tmp/liver_copy_number.csv", index=False)
+            copy_number_df = map_copy_number(copy_number_data = "/tmp/raw_copynum_data.csv", improve_id_data = "/tmp/liver_samples.csv", entrez_data = "/tmp/genes.csv")
+            copy_number_df.drop_duplicates(inplace=True)
+            copy_number_df.to_csv("/tmp/liver_copy_number.csv", index=False)
 
     if args.proteomics:
         if args.genes is None or args.genes=='':

coderbuild/novartis/02-omics-novartis.py

@@ -295,6 +295,7 @@ def map_mutations_novPDX(mutation_data, improve_id_data, entrez_data):
         else:
             print("Starting transcriptomics data.")
             transcriptomics_df_final = map_transcriptomics_novPDX(transcriptomics_data = "/tmp/raw_rnaseq_data.csv", improve_id_data = "/tmp/novartis_samples.csv", entrez_data = "/tmp/genes.csv")
+            transcriptomics_df_final.drop_duplicates(inplace=True)
             transcriptomics_df_final.to_csv("/tmp/novartis_transcriptomics.csv", index=False)
 
     if args.mutations:
@@ -307,6 +308,7 @@ def map_mutations_novPDX(mutation_data, improve_id_data, entrez_data):
         else:
             print("Starting mutations data.")
             mutation_df_final = map_mutations_novPDX(mutation_data = "/tmp/raw_mutation_data.csv", improve_id_data = "/tmp/novartis_samples.csv", entrez_data = "/tmp/genes.csv")
+            mutation_df_final.drop_duplicates(inplace=True)
             mutation_df_final.to_csv("/tmp/novartis_mutations.csv", index=False)
 
     if args.copy_number:

coderbuild/pancreatic/02a-getPancreaticDataFromSynapse.py

@@ -165,6 +165,7 @@ def main():
             res = parseMutFile(path,sampid, genes)
             alldats.append(res)
         newmut = pd.concat(alldats)
+        newmut.drop_duplicates(inplace=True)
         newmut.to_csv("/tmp/pancreatic_mutations.csv.gz",compression='gzip',index=False)
     #pd.DataFrame(missingsamples).to_csv('missing.csv',index=False,quoting=None,header=False)
 if __name__=='__main__':

coderbuild/utils/pubchem_retrieval.py

@@ -441,6 +441,7 @@ def update_dataframe_and_write_tsv(unique_names,
         final_df = pd.DataFrame(columns=combined.columns)
 
     # --- 10) write final filtered output ---
+    final_df.drop_duplicates(inplace=True)
     final_df.to_csv(output_filename, sep="\t", index=False)
 
     if os.path.exists(temp_file):