From 4d2d374d3a7ebfac80b11fb43635764426084ada Mon Sep 17 00:00:00 2001 From: lcoombe Date: Fri, 20 Sep 2024 11:54:37 -0700 Subject: [PATCH 1/7] Unpin python in azure-pipelines.yml --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f9a2ccf..da14bbf 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,7 +18,7 @@ jobs: displayName: Create Anaconda environment - script: | source activate ntroot_CI - conda install --yes --name ntroot_CI -c conda-forge -c bioconda python=3.9 mamba + conda install --yes --name ntroot_CI -c conda-forge -c bioconda python mamba mamba install --yes -c conda-forge -c bioconda meson ninja snakemake perl 'ntedit>=2.0.1' samtools bedtools displayName: Install Conda packages @@ -47,7 +47,7 @@ jobs: displayName: Create Anaconda environment - script: | source activate ntroot_CI - conda install --yes --name ntroot_CI -c conda-forge -c bioconda python=3.9 mamba + conda install --yes --name ntroot_CI -c conda-forge -c bioconda python mamba mamba install --yes -c conda-forge -c bioconda snakemake perl 'ntedit>=2.0.1' samtools bedtools displayName: Install Conda packages From 4d55927d487ece047f7701d950ba53b2185164cf Mon Sep 17 00:00:00 2001 From: lcoombe Date: Fri, 20 Sep 2024 12:02:31 -0700 Subject: [PATCH 2/7] Change single character keys in snakemake config to multi-character --- ntroot | 10 +++++----- ntroot_run_pipeline.smk | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ntroot b/ntroot index 4f45196..1b1ae39 100755 --- a/ntroot +++ b/ntroot @@ -111,7 +111,7 @@ def main(): smk_rule = "ntroot_input_vcf" command = f"snakemake -s {base_dir}/ntroot_run_pipeline.smk {smk_rule} --nolock -p --cores {args.t} " \ - f"--config reference={args.reference} t={args.t} " \ + f"--config reference={args.reference} threads={args.t} " \ f"tile_size={args.tile} " if args.genome: @@ -135,21 +135,21 @@ def main(): f"\t-z {args.z}", f"\t-j {args.j}", f"\t-Y {args.Y}"]) - command += f" k={args.k} z={args.z} j={args.j} Y={args.Y}" + command += f" kmer={args.k} z_param={args.z} j_param={args.j} Y_param={args.Y}" if args.lai: intro_string.append("\t--lai") if args.verbose: intro_string.append("\t-v") - command += " v=1" + command += " verbose=1" else: - command += " v=0" + command += " verbose=0" if not os.path.isfile(args.l): raise FileNotFoundError(f"VCF file {args.l} not found") intro_string.append(f"\t-l {args.l}") - command += f" l={args.l}" + command += f" l_param={args.l}" print("\n".join(intro_string), flush=True) diff --git a/ntroot_run_pipeline.smk b/ntroot_run_pipeline.smk index 581c8e4..819edd7 100644 --- a/ntroot_run_pipeline.smk +++ b/ntroot_run_pipeline.smk @@ -11,22 +11,22 @@ onsuccess: reference=config["reference"] draft_base = os.path.basename(os.path.realpath(reference)) reads_prefix=config["reads"] if "reads" in config else "" -k=config["k"] if "k" in config else None +k=config["kmer"] if "kmer" in config else None genomes = config["genomes"] if "genomes" in config else "" genome_prefix = ".".join([os.path.basename(os.path.realpath(genome)).removesuffix(".fa").removesuffix(".fasta").removesuffix(".fna") for genome in genomes]) # Common parameters -t = config["t"] if "t" in config else 4 -b = config["b"] + "_" if "b" in config and config["b"] != "" else "" +t = config["threads"] if "threads" in config else 4 +b = config["b_prefix"] + "_" if "b_prefix" in config and config["b_prefix"] != "" else "" # ntEdit parameters -z = config["z"] if "z" in config else 100 -v = config["v"] if "v" in config else 0 -j = config["j"] if "j" in config else 3 -Y = config["Y"] if "Y" in config else 0.55 -l = config["l"] if "l" in config else "" +z = config["z_param"] if "z_param" in config else 100 +v = config["verbose"] if "verbose" in config else 0 +j = config["j_param"] if "j_param" in config else 3 +Y = config["Y_param"] if "Y_param" in config else 0.55 +l = config["l_vcf"] if "l_vcf" in config else "" # Ancestry inference parameters tile_size = config["tile_size"] if "tile_size" in config else 5000000 From 04aac567bbeffc0be2af73852ecdf2fd85d4f0da Mon Sep 17 00:00:00 2001 From: lcoombe Date: Fri, 20 Sep 2024 12:08:21 -0700 Subject: [PATCH 3/7] Fix -l parameter for snakemake --- ntroot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ntroot b/ntroot index 1b1ae39..09c084c 100755 --- a/ntroot +++ b/ntroot @@ -149,7 +149,7 @@ def main(): if not os.path.isfile(args.l): raise FileNotFoundError(f"VCF file {args.l} not found") intro_string.append(f"\t-l {args.l}") - command += f" l_param={args.l}" + command += f" l_vcf={args.l}" print("\n".join(intro_string), flush=True) From 9d854434715a10ae247b04b8dbb3bc9821405bf4 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Fri, 20 Sep 2024 12:17:35 -0700 Subject: [PATCH 4/7] Temporarily build ntEdit from source --- azure-pipelines.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index da14bbf..35d7004 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,12 +20,22 @@ jobs: source activate ntroot_CI conda install --yes --name ntroot_CI -c conda-forge -c bioconda python mamba mamba install --yes -c conda-forge -c bioconda meson ninja snakemake perl 'ntedit>=2.0.1' samtools bedtools + mamba install --yes -c conda-forge -c bioconda libcxx llvm meson ninja btllib zlib boost cmake compilers displayName: Install Conda packages + + - script: | + git clone https://github.com/bcgsc/ntEdit.git + cd ntEdit + meson setup build --prefix=$(pwd)/ntedit_build + cd build && ninja install + displayName: Build ntEdit (until 2.0.3 updated in conda) - script: | source activate ntroot_CI + export PATH=$(pwd)/ntEdit/ntedit_build/bin:$PATH export PATH=$(pwd):$PATH which ntroot + which ntedit cd demo ./run_ntroot_demo.sh displayName: Run ntRoot test From 8835c40fa0ec2d145c19ef4cf4eb914f7ab48592 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Fri, 20 Sep 2024 12:20:22 -0700 Subject: [PATCH 5/7] Activate conda env in ntEdit source build --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 35d7004..3882dae 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,6 +24,7 @@ jobs: displayName: Install Conda packages - script: | + source activate ntroot_CI git clone https://github.com/bcgsc/ntEdit.git cd ntEdit meson setup build --prefix=$(pwd)/ntedit_build From 2fc0d3aa453bba2c87bf8bddf2d0b4b7645293f3 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Fri, 20 Sep 2024 12:24:48 -0700 Subject: [PATCH 6/7] ntEdit source build for macOS CI added --- azure-pipelines.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3882dae..fc6dd8c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -60,7 +60,16 @@ jobs: source activate ntroot_CI conda install --yes --name ntroot_CI -c conda-forge -c bioconda python mamba mamba install --yes -c conda-forge -c bioconda snakemake perl 'ntedit>=2.0.1' samtools bedtools + mamba install --yes -c conda-forge -c bioconda libcxx llvm meson ninja btllib zlib boost cmake compilers displayName: Install Conda packages + + - script: | + source activate ntroot_CI + git clone https://github.com/bcgsc/ntEdit.git + cd ntEdit + meson setup build --prefix=$(pwd)/ntedit_build + cd build && ninja install + displayName: Build ntEdit (until 2.0.3 updated in conda) - script: | source activate ntroot_CI From 18026d150ea42740ec4f9979a437e6aeb18ad4b8 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Fri, 20 Sep 2024 13:00:11 -0700 Subject: [PATCH 7/7] Adjust PATH for mac ntEdit source build --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fc6dd8c..8e9d0dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -73,8 +73,10 @@ jobs: - script: | source activate ntroot_CI + export PATH=$(pwd)/ntEdit/ntedit_build/bin:$PATH export PATH=$(pwd):$PATH which ntroot + which ntedit cd demo ./run_ntroot_demo.sh displayName: Run ntRoot test