diff --git a/doc/analyze.rst b/doc/analyze.rst index 2a97c766f..d16107b0f 100644 --- a/doc/analyze.rst +++ b/doc/analyze.rst @@ -295,18 +295,18 @@ As in the case of ``analyze amplicon``, required option ``--starting-material`` Resuming execution / re-analyzing --------------------------------- -MiXCR allows to continue aborted execution or re-analyze existing data with updated parameters, without complete re-processing of each of pipeline steps. This is possible with ``--resume`` option. For example, suppose the analysis was performed with the default options: +MiXCR allows to continue aborted execution or re-analyze existing data with updated parameters, without complete re-processing of each of pipeline steps. This is possible with ``--overwrite-if-required`` option. For example, suppose the analysis was performed with the default options: :: mixcr analyze shotgun --species hs --starting-material rna data_R1.fastq data_R2.fastq analysis_name -Now to re-analyze the data with updated options for :ref:`assemble `, one can pass ``--resume`` option in order to avoid unnecessary invocation of ``align``, ``assemblePartial`` and ``extend``: +Now to re-analyze the data with updated options for :ref:`assemble `, one can pass ``--overwrite-if-required`` option in order to avoid unnecessary invocation of ``align``, ``assemblePartial`` and ``extend``: :: - mixcr analyze shotgun --force --resume --species hs --starting-material rna --assemble "-ObadQualityThreshold=0" data_R1.fastq data_R2.fastq analysis_name + mixcr analyze shotgun --overwrite-if-required --species hs --starting-material rna --assemble "-ObadQualityThreshold=0" data_R1.fastq data_R2.fastq analysis_name This way, the previous results of :ref:`align `, :ref:`assemblePartial ` and :ref:`extend ` will be used, while :ref:`assembly step ` will be re-executed. diff --git a/itests.sh b/itests.sh index d3679a8d3..7d7d54785 100755 --- a/itests.sh +++ b/itests.sh @@ -106,4 +106,10 @@ if [[ $run_tests == true ]]; then echo "Running test case 1" mixcr align -s hs -OvParameters.geneFeatureToAlign=VGeneWithP test_R1.fastq test_R2.fastq case1.vdjca mixcr assemble case1.vdjca case1.clns + + echo "Running test case 2" + mixcr analyze shotgun -f --species hs --contig-assembly --impute-germline-on-export --starting-material rna test_R1.fastq test_R2.fastq case2 + + echo "Running test case 3" + mixcr analyze amplicon --receptor-type tra --impute-germline-on-export -s hs --starting-material rna --contig-assembly --5-end v-primers --3-end j-primers --adapters no-adapters test_R1.fastq test_R2.fastq case3 fi diff --git a/mixcr_completion b/mixcr_completion index 4ec0ee83b..3f5c6029b 100644 --- a/mixcr_completion +++ b/mixcr_completion @@ -1,4 +1,3 @@ - #!/usr/bin/env bash # # mixcr Bash Completion @@ -79,11 +78,11 @@ function _complete_mixcr() { CMDS2=(align) CMDS3=(assemble) CMDS4=(assembleContigs) - CMDS5=(exportClones) - CMDS6=(assemblePartial) - CMDS7=(extend) - CMDS8=(exportAlignments) - CMDS9=(exportAlignmentsPretty) + CMDS5=(assemblePartial) + CMDS6=(extend) + CMDS7=(exportAlignments) + CMDS8=(exportAlignmentsPretty) + CMDS9=(exportClones) CMDS10=(exportClonesPretty) CMDS11=(exportReadsForClones) CMDS12=(exportReads) @@ -119,11 +118,11 @@ function _complete_mixcr() { ArrContains COMP_WORDS CMDS12 && { _picocli_mixcr_exportReads; return $?; } ArrContains COMP_WORDS CMDS11 && { _picocli_mixcr_exportReadsForClones; return $?; } ArrContains COMP_WORDS CMDS10 && { _picocli_mixcr_exportClonesPretty; return $?; } - ArrContains COMP_WORDS CMDS9 && { _picocli_mixcr_exportAlignmentsPretty; return $?; } - ArrContains COMP_WORDS CMDS8 && { _picocli_mixcr_exportAlignments; return $?; } - ArrContains COMP_WORDS CMDS7 && { _picocli_mixcr_extend; return $?; } - ArrContains COMP_WORDS CMDS6 && { _picocli_mixcr_assemblePartial; return $?; } - ArrContains COMP_WORDS CMDS5 && { _picocli_mixcr_exportClones; return $?; } + ArrContains COMP_WORDS CMDS9 && { _picocli_mixcr_exportClones; return $?; } + ArrContains COMP_WORDS CMDS8 && { _picocli_mixcr_exportAlignmentsPretty; return $?; } + ArrContains COMP_WORDS CMDS7 && { _picocli_mixcr_exportAlignments; return $?; } + ArrContains COMP_WORDS CMDS6 && { _picocli_mixcr_extend; return $?; } + ArrContains COMP_WORDS CMDS5 && { _picocli_mixcr_assemblePartial; return $?; } ArrContains COMP_WORDS CMDS4 && { _picocli_mixcr_assembleContigs; return $?; } ArrContains COMP_WORDS CMDS3 && { _picocli_mixcr_assemble; return $?; } ArrContains COMP_WORDS CMDS2 && { _picocli_mixcr_align; return $?; } @@ -140,7 +139,7 @@ function _picocli_mixcr() { CURR_WORD=${COMP_WORDS[COMP_CWORD]} PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} - COMMANDS="help analyze align assemble assembleContigs exportClones assemblePartial extend exportAlignments exportAlignmentsPretty exportClonesPretty exportReadsForClones exportReads mergeAlignments filterAlignments sortAlignments alignmentsDiff clonesDiff alignmentsStat listLibraries versionInfo pipelineInfo slice info" + COMMANDS="help analyze align assemble assembleContigs assemblePartial extend exportAlignments exportAlignmentsPretty exportClones exportClonesPretty exportReadsForClones exportReads mergeAlignments filterAlignments sortAlignments alignmentsDiff clonesDiff alignmentsStat listLibraries versionInfo pipelineInfo slice info" FLAG_OPTS="-h --help -v --version -h --help" ARG_OPTS="" @@ -192,8 +191,8 @@ function _picocli_mixcr_align() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --resume -d --no-merge --write-all --buffers -a --save-description -g --save-reads" - ARG_OPTS="-s --species -r --report --json-report -b --library -p --parameters -O --not-aligned-R1 --not-aligned-R2 -t --threads -c --chains -n --limit" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --overwrite-if-required -d --no-merge --write-all --buffers -a --save-description -g --save-reads" + ARG_OPTS="-s --species -r --report --json-report -b --library -p --parameters -O --not-aligned-R1 --not-aligned-R2 -n --limit -t --threads -c --chains" compopt +o default @@ -222,13 +221,13 @@ function _picocli_mixcr_align() { --not-aligned-R2) return ;; - -t|--threads) + -n|--limit) return ;; - -c|--chains) + -t|--threads) return ;; - -n|--limit) + -c|--chains) return ;; esac @@ -247,8 +246,8 @@ function _picocli_mixcr_assemble() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --resume --buffers -a --write-alignments" - ARG_OPTS="-p --parameters -r --report --json-report -e --events -O -t --threads" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --overwrite-if-required --buffers -a --write-alignments" + ARG_OPTS="-p --parameters -r --report --json-report -O -t --threads" compopt +o default @@ -262,9 +261,6 @@ function _picocli_mixcr_assemble() { --json-report) return ;; - -e|--events) - return - ;; -O) return ;; @@ -287,7 +283,7 @@ function _picocli_mixcr_assembleContigs() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --resume" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --overwrite-if-required" ARG_OPTS="-O -r --report --debug-report --json-report -t --threads" compopt +o default @@ -317,94 +313,6 @@ function _picocli_mixcr_assembleContigs() { fi } -# Generates completions for the options and subcommands of the `exportClones` subcommand. -function _picocli_mixcr_exportClones() { - # Get completion data - CURR_WORD=${COMP_WORDS[COMP_CWORD]} - PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} - - COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force -v --with-spaces -lf --list-fields -s --no-spaces -o --filter-out-of-frames -t --filter-stops -targets -vHit -dHit -jHit -cHit -vGene -dGene -jGene -cGene -vFamily -dFamily -jFamily -cFamily -vHitScore -dHitScore -jHitScore -cHitScore -vHitsWithScore -dHitsWithScore -jHitsWithScore -cHitsWithScore -vHits -dHits -jHits -cHits -vGenes -dGenes -jGenes -cGenes -vFamilies -dFamilies -jFamilies -cFamilies -vAlignment -dAlignment -jAlignment -cAlignment -vAlignments -dAlignments -jAlignments -cAlignments -defaultAnchorPoints -cloneId -count -fraction -sequence -quality -vIdentityPercents -dIdentityPercents -jIdentityPercents -cIdentityPercents -vBestIdentityPercent -dBestIdentityPercent -jBestIdentityPercent -cBestIdentityPercent -chains -topChains" - ARG_OPTS="-c --chains -p --preset -pf --preset-file -n --limit -q --minimal-clone-fraction -m --minimal-clone-count -nFeature -qFeature -aaFeature -nFeatureImputed -aaFeatureImputed -minFeatureQuality -avrgFeatureQuality -lengthOf -nMutations -nMutationsRelative -aaMutations -aaMutationsRelative -mutationsDetailed -mutationsDetailedRelative -positionInReferenceOf -positionOf" - - compopt +o default - - case ${PREV_WORD} in - -c|--chains) - return - ;; - -p|--preset) - return - ;; - -pf|--preset-file) - return - ;; - -n|--limit) - return - ;; - -q|--minimal-clone-fraction) - return - ;; - -m|--minimal-clone-count) - return - ;; - -nFeature) - return - ;; - -qFeature) - return - ;; - -aaFeature) - return - ;; - -nFeatureImputed) - return - ;; - -aaFeatureImputed) - return - ;; - -minFeatureQuality) - return - ;; - -avrgFeatureQuality) - return - ;; - -lengthOf) - return - ;; - -nMutations) - return - ;; - -nMutationsRelative) - return - ;; - -aaMutations) - return - ;; - -aaMutationsRelative) - return - ;; - -mutationsDetailed) - return - ;; - -mutationsDetailedRelative) - return - ;; - -positionInReferenceOf) - return - ;; - -positionOf) - return - ;; - esac - - if [[ "${CURR_WORD}" == -* ]]; then - COMPREPLY=( $(compgen -W "${FLAG_OPTS} ${ARG_OPTS}" -- ${CURR_WORD}) ) - else - COMPREPLY=( $(compgen -W "${COMMANDS}" -- ${CURR_WORD}) ) - fi -} - # Generates completions for the options and subcommands of the `assemblePartial` subcommand. function _picocli_mixcr_assemblePartial() { # Get completion data @@ -412,7 +320,7 @@ function _picocli_mixcr_assemblePartial() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --resume -o --overlapped-only -d --drop-partial" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --overwrite-if-required -o --overlapped-only -d --drop-partial" ARG_OPTS="-O -r --report --json-report" compopt +o default @@ -443,7 +351,7 @@ function _picocli_mixcr_extend() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --resume" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --overwrite-if-required" ARG_OPTS="-c --chains -r --report --json-report -q --quality --v-anchor --j-anchor --min-v-score --min-j-score -t --threads" compopt +o default @@ -492,7 +400,7 @@ function _picocli_mixcr_exportAlignments() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force -v --with-spaces -lf --list-fields -s --no-spaces -targets -vHit -dHit -jHit -cHit -vGene -dGene -jGene -cGene -vFamily -dFamily -jFamily -cFamily -vHitScore -dHitScore -jHitScore -cHitScore -vHitsWithScore -dHitsWithScore -jHitsWithScore -cHitsWithScore -vHits -dHits -jHits -cHits -vGenes -dGenes -jGenes -cGenes -vFamilies -dFamilies -jFamilies -cFamilies -vAlignment -dAlignment -jAlignment -cAlignment -vAlignments -dAlignments -jAlignments -cAlignments -defaultAnchorPoints -readId -readIds -sequence -quality -descrR1 -descrR2 -descrsR1 -descrsR2 -readHistory -cloneId -cloneIdWithMappingType -vIdentityPercents -dIdentityPercents -jIdentityPercents -cIdentityPercents -vBestIdentityPercent -dBestIdentityPercent -jBestIdentityPercent -cBestIdentityPercent -chains -topChains" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force -v --with-spaces -lf --list-fields -s --no-spaces -targets -vHit -dHit -jHit -cHit -vGene -dGene -jGene -cGene -vFamily -dFamily -jFamily -cFamily -vHitScore -dHitScore -jHitScore -cHitScore -vHitsWithScore -dHitsWithScore -jHitsWithScore -cHitsWithScore -vHits -dHits -jHits -cHits -vGenes -dGenes -jGenes -cGenes -vFamilies -dFamilies -jFamilies -cFamilies -vAlignment -dAlignment -jAlignment -cAlignment -vAlignments -dAlignments -jAlignments -cAlignments -defaultAnchorPoints -readId -readIds -sequence -quality -descrR1 -descrR2 -descrsR1 -descrsR2 -readHistory -cloneId -cloneIdWithMappingType -vIdentityPercents -dIdentityPercents -jIdentityPercents -cIdentityPercents -vBestIdentityPercent -dBestIdentityPercent -jBestIdentityPercent -cBestIdentityPercent -chains -topChains" ARG_OPTS="-c --chains -p --preset -pf --preset-file -n --limit -nFeature -qFeature -aaFeature -nFeatureImputed -aaFeatureImputed -minFeatureQuality -avrgFeatureQuality -lengthOf -nMutations -nMutationsRelative -aaMutations -aaMutationsRelative -mutationsDetailed -mutationsDetailedRelative -positionInReferenceOf -positionOf" compopt +o default @@ -574,7 +482,7 @@ function _picocli_mixcr_exportAlignmentsPretty() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force -t --top -a --gene -d --descriptions" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force -t --top -a --gene -d --descriptions" ARG_OPTS="-b --limit-before -n --limit -c --chains -s --skip -e --cdr3-equals -g --feature -r --read-contains --filter -i --read-ids" compopt +o default @@ -616,6 +524,94 @@ function _picocli_mixcr_exportAlignmentsPretty() { fi } +# Generates completions for the options and subcommands of the `exportClones` subcommand. +function _picocli_mixcr_exportClones() { + # Get completion data + CURR_WORD=${COMP_WORDS[COMP_CWORD]} + PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} + + COMMANDS="" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force -v --with-spaces -lf --list-fields -s --no-spaces -o --filter-out-of-frames -t --filter-stops -targets -vHit -dHit -jHit -cHit -vGene -dGene -jGene -cGene -vFamily -dFamily -jFamily -cFamily -vHitScore -dHitScore -jHitScore -cHitScore -vHitsWithScore -dHitsWithScore -jHitsWithScore -cHitsWithScore -vHits -dHits -jHits -cHits -vGenes -dGenes -jGenes -cGenes -vFamilies -dFamilies -jFamilies -cFamilies -vAlignment -dAlignment -jAlignment -cAlignment -vAlignments -dAlignments -jAlignments -cAlignments -defaultAnchorPoints -cloneId -count -fraction -sequence -quality -vIdentityPercents -dIdentityPercents -jIdentityPercents -cIdentityPercents -vBestIdentityPercent -dBestIdentityPercent -jBestIdentityPercent -cBestIdentityPercent -chains -topChains" + ARG_OPTS="-c --chains -p --preset -pf --preset-file -n --limit -q --minimal-clone-fraction -m --minimal-clone-count -nFeature -qFeature -aaFeature -nFeatureImputed -aaFeatureImputed -minFeatureQuality -avrgFeatureQuality -lengthOf -nMutations -nMutationsRelative -aaMutations -aaMutationsRelative -mutationsDetailed -mutationsDetailedRelative -positionInReferenceOf -positionOf" + + compopt +o default + + case ${PREV_WORD} in + -c|--chains) + return + ;; + -p|--preset) + return + ;; + -pf|--preset-file) + return + ;; + -n|--limit) + return + ;; + -q|--minimal-clone-fraction) + return + ;; + -m|--minimal-clone-count) + return + ;; + -nFeature) + return + ;; + -qFeature) + return + ;; + -aaFeature) + return + ;; + -nFeatureImputed) + return + ;; + -aaFeatureImputed) + return + ;; + -minFeatureQuality) + return + ;; + -avrgFeatureQuality) + return + ;; + -lengthOf) + return + ;; + -nMutations) + return + ;; + -nMutationsRelative) + return + ;; + -aaMutations) + return + ;; + -aaMutationsRelative) + return + ;; + -mutationsDetailed) + return + ;; + -mutationsDetailedRelative) + return + ;; + -positionInReferenceOf) + return + ;; + -positionOf) + return + ;; + esac + + if [[ "${CURR_WORD}" == -* ]]; then + COMPREPLY=( $(compgen -W "${FLAG_OPTS} ${ARG_OPTS}" -- ${CURR_WORD}) ) + else + COMPREPLY=( $(compgen -W "${COMMANDS}" -- ${CURR_WORD}) ) + fi +} + # Generates completions for the options and subcommands of the `exportClonesPretty` subcommand. function _picocli_mixcr_exportClonesPretty() { # Get completion data @@ -623,7 +619,7 @@ function _picocli_mixcr_exportClonesPretty() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force" ARG_OPTS="-b --limitBefore -n --limit -i --clone-ids -s --skip -c --chains -e --cdr3-equals -r --clonal-sequence-contains" compopt +o default @@ -666,7 +662,7 @@ function _picocli_mixcr_exportReadsForClones() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force -s --separate" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force -s --separate" ARG_OPTS="--id" compopt +o default @@ -691,7 +687,7 @@ function _picocli_mixcr_exportReads() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force" ARG_OPTS="" if [[ "${CURR_WORD}" == -* ]]; then @@ -708,7 +704,7 @@ function _picocli_mixcr_mergeAlignments() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --resume" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --overwrite-if-required" ARG_OPTS="" if [[ "${CURR_WORD}" == -* ]]; then @@ -725,7 +721,7 @@ function _picocli_mixcr_filterAlignments() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --resume -x --chimeras-only" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --overwrite-if-required -x --chimeras-only" ARG_OPTS="-c --chains -g --contains-feature -e --cdr3-equals -i --read-ids -n --limit" compopt +o default @@ -762,7 +758,7 @@ function _picocli_mixcr_sortAlignments() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --resume" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --overwrite-if-required" ARG_OPTS="" if [[ "${CURR_WORD}" == -* ]]; then @@ -779,7 +775,7 @@ function _picocli_mixcr_alignmentsDiff() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force" ARG_OPTS="-o1 --only-in-first -o2 --only-in-second -d1 --diff-from-first -d2 --diff-from-second -g --gene-feature -l --top-hits-level" compopt +o default @@ -819,7 +815,7 @@ function _picocli_mixcr_clonesDiff() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force -v -j -c" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force -v -j -c" ARG_OPTS="" if [[ "${CURR_WORD}" == -* ]]; then @@ -904,7 +900,7 @@ function _picocli_mixcr_slice() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --resume" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --overwrite-if-required" ARG_OPTS="-i --id" compopt +o default @@ -963,7 +959,7 @@ function _picocli_mixcr_analyze_amplicon() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --impute-germline-on-export --only-productive --contig-assembly --resume --do-not-extend-alignments" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --impute-germline-on-export --only-productive --contig-assembly --do-not-extend-alignments" ARG_OPTS="-s --species --starting-material -r --report --align --assemblePartial --extend --assemble --assembleContigs --export --assemble-partial-rounds --receptor-type --5-end --3-end --adapters --region-of-interest" startingMaterial_OPTION_ARGS="rna dna" # _StartingMaterial values chains_OPTION_ARGS="tcr bcr xcr tra trb trd trg igh igk igl" # --receptor-type values @@ -1040,7 +1036,7 @@ function _picocli_mixcr_analyze_shotgun() { PREV_WORD=${COMP_WORDS[COMP_CWORD-1]} COMMANDS="" - FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force --impute-germline-on-export --only-productive --contig-assembly --resume --do-not-extend-alignments" + FLAG_OPTS="-h --help -nw --no-warnings --verbose -f --force-overwrite --force --impute-germline-on-export --only-productive --contig-assembly --do-not-extend-alignments" ARG_OPTS="-s --species --starting-material -r --report --align --assemblePartial --extend --assemble --assembleContigs --export --assemble-partial-rounds --receptor-type" startingMaterial_OPTION_ARGS="rna dna" # _StartingMaterial values chains_OPTION_ARGS="tcr bcr xcr tra trb trd trg igh igk igl" # --receptor-type values @@ -1099,4 +1095,7 @@ function _picocli_mixcr_analyze_shotgun() { # current word on the command line. # The `-o default` option means that if the function generated no matches, the # default Bash completions and the Readline default filename completions are performed. -complete -F _complete_mixcr -o default mixcr mixcr.sh mixcr.bash \ No newline at end of file +complete -F _complete_mixcr -o default mixcr mixcr.sh mixcr.bash + + +Process finished with exit code 0 diff --git a/src/main/java/com/milaboratory/mixcr/basictypes/AlignmentsIO.java b/src/main/java/com/milaboratory/mixcr/basictypes/AlignmentsIO.java index 2b68f4973..277a143ff 100644 --- a/src/main/java/com/milaboratory/mixcr/basictypes/AlignmentsIO.java +++ b/src/main/java/com/milaboratory/mixcr/basictypes/AlignmentsIO.java @@ -59,7 +59,7 @@ * [ 4 bytes : int : number of alignments ] * [ 4 bytes : int : rawDataSize ] * [ 4 bytes : int : compressedDataSize ] - * [ 4 bytes : int : checksum for raw data ] + * [ 4 bytes : int : checksum for the raw data ] * * Data: * [ dataSize bytes ] (compressed, if bit1 of header is 1; uncompressed, if bit1 is 0 ) diff --git a/src/main/java/com/milaboratory/mixcr/basictypes/ClnAReader.java b/src/main/java/com/milaboratory/mixcr/basictypes/ClnAReader.java index 71de421c5..d3423acb2 100644 --- a/src/main/java/com/milaboratory/mixcr/basictypes/ClnAReader.java +++ b/src/main/java/com/milaboratory/mixcr/basictypes/ClnAReader.java @@ -62,8 +62,8 @@ * Reader of CLNA file format. */ public final class ClnAReader implements - PipelineConfigurationReader, - AutoCloseable { + PipelineConfigurationReader, + AutoCloseable { public static final int DEFAULT_CHUNK_SIZE = 262144; final int chunkSize; /** @@ -134,7 +134,7 @@ public ClnAReader(Path path, VDJCLibraryRegistry libraryRegistry, int chunkSize) buf.flip(); buf.limit(16); - long fSize = channel.size(); + long fSize = channel.size() - IOUtil.END_MAGIC_LENGTH; channel.read(buf, fSize - 16); buf.flip(); this.firstClonePosition = buf.getLong(); diff --git a/src/main/java/com/milaboratory/mixcr/basictypes/ClnAWriter.java b/src/main/java/com/milaboratory/mixcr/basictypes/ClnAWriter.java index 3a7c7609a..16b070285 100644 --- a/src/main/java/com/milaboratory/mixcr/basictypes/ClnAWriter.java +++ b/src/main/java/com/milaboratory/mixcr/basictypes/ClnAWriter.java @@ -61,7 +61,7 @@ public final class ClnAWriter implements PipelineConfigurationWriter, CanReportProgressAndStage { static final String MAGIC_V3 = "MiXCR.CLNA.V03"; static final String MAGIC = MAGIC_V3; - static final int MAGIC_LENGTH = MAGIC.length(); + static final int MAGIC_LENGTH = MAGIC.length(); //14 /** * Will be used for alignments pre-sorting @@ -350,6 +350,9 @@ public String getStatus() { output.writeLong(positionOfFirstClone); output.writeLong(indexBeginOffset); + // Writing end-magic as a file integrity sign + output.write(IOUtil.getEndMagicBytes()); + // Setting finished flag (will stop progress reporting) finished = true; } diff --git a/src/main/java/com/milaboratory/mixcr/basictypes/ClnsWriter.java b/src/main/java/com/milaboratory/mixcr/basictypes/ClnsWriter.java index d3e930232..1d2cbd192 100644 --- a/src/main/java/com/milaboratory/mixcr/basictypes/ClnsWriter.java +++ b/src/main/java/com/milaboratory/mixcr/basictypes/ClnsWriter.java @@ -16,8 +16,8 @@ * */ public class ClnsWriter implements PipelineConfigurationWriter, - CanReportProgressAndStage, - Closeable { + CanReportProgressAndStage, + Closeable { static final String MAGIC_V7 = "MiXCR.CLNS.V07"; static final String MAGIC = MAGIC_V7; static final int MAGIC_LENGTH = 14; @@ -87,6 +87,9 @@ public void write() { output.writeObject(clone); ++current; } + + // Writing end-magic as a file integrity sign + output.write(IOUtil.getEndMagicBytes()); } @Override diff --git a/src/main/java/com/milaboratory/mixcr/basictypes/CloneSetIO.java b/src/main/java/com/milaboratory/mixcr/basictypes/CloneSetIO.java index 2b304c54f..9d8dd54d5 100644 --- a/src/main/java/com/milaboratory/mixcr/basictypes/CloneSetIO.java +++ b/src/main/java/com/milaboratory/mixcr/basictypes/CloneSetIO.java @@ -34,6 +34,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.util.Objects; public final class CloneSetIO { public static CloneSet read(String file) throws IOException { @@ -49,7 +50,7 @@ public static CloneSet read(String file, VDJCLibraryRegistry libraryRegistry) th } public static CloneSet read(File file, VDJCLibraryRegistry libraryRegistry) throws IOException { - switch (IOUtil.detectFilType(file)) { + switch (Objects.requireNonNull(IOUtil.getFileInfo(file)).fileType) { case ClnA: try (ClnAReader r = new ClnAReader(file.toPath(), libraryRegistry)) { return r.readCloneSet(); diff --git a/src/main/java/com/milaboratory/mixcr/basictypes/IOUtil.java b/src/main/java/com/milaboratory/mixcr/basictypes/IOUtil.java index ecfd1d702..aa463697c 100644 --- a/src/main/java/com/milaboratory/mixcr/basictypes/IOUtil.java +++ b/src/main/java/com/milaboratory/mixcr/basictypes/IOUtil.java @@ -39,11 +39,29 @@ import io.repseq.core.VDJCLibraryRegistry; import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Objects; public class IOUtil { + public static final int BEGIN_MAGIC_LENGTH = 14; + public static final int BEGIN_MAGIC_LENGTH_SHORT = 10; + + public static final String END_MAGIC = "#MiXCR.File.End#"; + private static final byte[] END_MAGIC_BYTES = END_MAGIC.getBytes(StandardCharsets.US_ASCII); + public static final int END_MAGIC_LENGTH = END_MAGIC_BYTES.length; + + public static byte[] getEndMagicBytes() { + return END_MAGIC_BYTES.clone(); + } + public static void writeAndRegisterGeneReferences(PrimitivO output, List genes, HasFeatureToAlign featuresToAlign) { // Writing gene ids @@ -143,34 +161,91 @@ public static OutputStream createOS(CompressionType ct, OutputStream os) throws else return ct.createOutputStream(os, 65536); } - public enum MiXCRFileType { - Clns, ClnA, VDJCA - } - - public static MiXCRFileType detectFilType(String file) { - return detectFilType(new File(file)); + public static MiXCRFileInfo getFileInfo(String fileName) { + return getFileInfo(new File(fileName)); } - public static MiXCRFileType detectFilType(File file) { - CompressionType ct = CompressionType.detectCompressionType(file); + public static MiXCRFileInfo getFileInfo(File file) { try { - try (InputStream reader = ct.createInputStream(new FileInputStream(file))) { - byte[] data = new byte[10]; - reader.read(data); - String magic = new String(data, StandardCharsets.US_ASCII); - switch (magic) { + Path path = file.toPath(); + + if (!Files.isRegularFile(path)) + return null; + + try (FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { + if (channel.size() < BEGIN_MAGIC_LENGTH + END_MAGIC_LENGTH) + return null; + + byte[] beginMagic = new byte[BEGIN_MAGIC_LENGTH]; + channel.read(ByteBuffer.wrap(beginMagic)); + String magicFull = new String(beginMagic, StandardCharsets.US_ASCII); + String magicShort = new String(beginMagic, 0, BEGIN_MAGIC_LENGTH_SHORT, StandardCharsets.US_ASCII); + MiXCRFileType type; + switch (magicShort) { case "MiXCR.VDJC": - return MiXCRFileType.VDJCA; + type = MiXCRFileType.VDJCA; + break; case "MiXCR.CLNS": - return MiXCRFileType.Clns; + type = MiXCRFileType.Clns; + break; case "MiXCR.CLNA": - return MiXCRFileType.ClnA; + type = MiXCRFileType.ClnA; + break; default: - throw new IllegalArgumentException("Not a MiXCR file"); + return null; } + + byte[] endMagic = new byte[END_MAGIC_LENGTH]; + channel.read(ByteBuffer.wrap(endMagic), channel.size() - END_MAGIC_LENGTH); + return new MiXCRFileInfo(type, magicFull, Arrays.equals(endMagic, getEndMagicBytes())); } } catch (IOException e) { throw new RuntimeException(e); } } + + public enum MiXCRFileType { + Clns, ClnA, VDJCA + } + + /** + * Represent MiXCR binary file information (type, magic bytes, and whether it passes integrity test) + */ + public static final class MiXCRFileInfo { + /** + * MiXCR file type, or null if not a MiXCR file format. + */ + public final MiXCRFileType fileType; + + /** + * Full magic bytes string. + */ + public final String fullMagic; + + /** + * True if file integrity check succeeded. The command generated the file was not prematurely interrupted. + */ + public final boolean valid; + + public MiXCRFileInfo(MiXCRFileType fileType, String fullMagic, boolean valid) { + this.fileType = fileType; + this.fullMagic = fullMagic; + this.valid = valid; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MiXCRFileInfo)) return false; + MiXCRFileInfo that = (MiXCRFileInfo) o; + return valid == that.valid && + fileType == that.fileType && + Objects.equals(fullMagic, that.fullMagic); + } + + @Override + public int hashCode() { + return Objects.hash(fileType, fullMagic, valid); + } + } } diff --git a/src/main/java/com/milaboratory/mixcr/basictypes/PipelineConfigurationReader.java b/src/main/java/com/milaboratory/mixcr/basictypes/PipelineConfigurationReader.java index 939519fb4..9c958b17e 100644 --- a/src/main/java/com/milaboratory/mixcr/basictypes/PipelineConfigurationReader.java +++ b/src/main/java/com/milaboratory/mixcr/basictypes/PipelineConfigurationReader.java @@ -11,19 +11,28 @@ public interface PipelineConfigurationReader { /** * Read pipeline configuration from file or return null */ - static PipelineConfiguration fromFileOrNull(String fileName) { + static PipelineConfiguration fromFileOrNull(String fileName, IOUtil.MiXCRFileInfo fileInfo) { + if (!fileInfo.valid) + return null; try { - return fromFile(fileName); + return fromFile(fileName, fileInfo); } catch (Throwable ignored) {} return null; } + static PipelineConfiguration fromFile(String fileName) { + IOUtil.MiXCRFileInfo fileInfo = IOUtil.getFileInfo(fileName); + if (!fileInfo.valid) + throw new RuntimeException("File " + fileName + " corrupted."); + return fromFile(fileName, fileInfo); + } + /** * Read pipeline configuration from file or throw exception */ - static PipelineConfiguration fromFile(String fileName) { + static PipelineConfiguration fromFile(String fileName, IOUtil.MiXCRFileInfo fileInfo) { try { - switch (IOUtil.detectFilType(fileName)) { + switch (fileInfo.fileType) { case VDJCA: try (VDJCAlignmentsReader reader = new VDJCAlignmentsReader(fileName)) { return reader.getPipelineConfiguration(); diff --git a/src/main/java/com/milaboratory/mixcr/basictypes/VDJCAlignmentsWriter.java b/src/main/java/com/milaboratory/mixcr/basictypes/VDJCAlignmentsWriter.java index 649b0ef84..aec7c13c6 100644 --- a/src/main/java/com/milaboratory/mixcr/basictypes/VDJCAlignmentsWriter.java +++ b/src/main/java/com/milaboratory/mixcr/basictypes/VDJCAlignmentsWriter.java @@ -222,7 +222,7 @@ public synchronized void close() { flushBlock(); writer.close(); // This will also write stream termination symbol/block to the stream - writerFactory.close(); + writerFactory.close(); // This blocks the thread until all workers flush their data to the underlying stream // [ numberOfProcessedReads : long ] byte[] footer = new byte[8]; @@ -230,6 +230,10 @@ public synchronized void close() { // Writing it as last piece of information in the stream AlignmentsIO.writeLongBE(numberOfProcessedReads, footer, 0); rawOutput.write(footer); + + // Writing end-magic as a file integrity sign + rawOutput.write(IOUtil.getEndMagicBytes()); + rawOutput.close(); closed = true; } diff --git a/src/main/java/com/milaboratory/mixcr/cli/ACommand.java b/src/main/java/com/milaboratory/mixcr/cli/ACommand.java index 6a01ad810..070bffb3a 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/ACommand.java +++ b/src/main/java/com/milaboratory/mixcr/cli/ACommand.java @@ -1,15 +1,12 @@ package com.milaboratory.mixcr.cli; -import picocli.CommandLine.ExecutionException; -import picocli.CommandLine.Model.CommandSpec; +import com.milaboratory.mixcr.basictypes.IOUtil; import picocli.CommandLine.Option; -import picocli.CommandLine.Spec; import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; /** * @@ -54,9 +51,13 @@ private void printWarn(String message) { /** Validate injected parameters and options */ public void validate() { - for (String in : getInputFiles()) + for (String in : getInputFiles()) { if (!new File(in).exists()) throwValidationException("ERROR: input file \"" + in + "\" does not exist.", false); + IOUtil.MiXCRFileInfo info = IOUtil.getFileInfo(in); + if (info != null && !info.valid) + throwValidationException("ERROR: input file \"" + in + "\" is corrupted.", false); + } } @Override diff --git a/src/main/java/com/milaboratory/mixcr/cli/ACommandWithOutput.java b/src/main/java/com/milaboratory/mixcr/cli/ACommandWithOutput.java index 1530598b5..2efb2a365 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/ACommandWithOutput.java +++ b/src/main/java/com/milaboratory/mixcr/cli/ACommandWithOutput.java @@ -6,11 +6,17 @@ /** A command which produce output files */ public abstract class ACommandWithOutput extends ACommand { - @Option(names = {"-f", "--force"}, + @Option(names = {"-f", "--force-overwrite"}, description = "Force overwrite of output file(s).") - public boolean force = false; + public boolean forceOverwrite = false; - public boolean isForceOverwrite() {return force;} + @Option(names = {"--force"}, hidden = true) + public void setForce(boolean value) { + if (value) { + warn("--force option is deprecated; use --force-overwrite instead."); + forceOverwrite = true; + } + } @Override public void validate() { @@ -22,7 +28,7 @@ public void validate() { /** Specifies behaviour in the case with output exists (default is to throw exception) */ public void handleExistenceOfOutputFile(String outFileName) { - if (!force) - throwValidationException("File \"" + outFileName + "\" already exists. Use -f option to overwrite it.", false); + if (!forceOverwrite) + throwValidationException("File \"" + outFileName + "\" already exists. Use -f / --force-overwrite option to overwrite it.", false); } } diff --git a/src/main/java/com/milaboratory/mixcr/cli/ACommandWithResume.java b/src/main/java/com/milaboratory/mixcr/cli/ACommandWithSmartOverwrite.java similarity index 61% rename from src/main/java/com/milaboratory/mixcr/cli/ACommandWithResume.java rename to src/main/java/com/milaboratory/mixcr/cli/ACommandWithSmartOverwrite.java index 119e6bee2..bc21f9b93 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/ACommandWithResume.java +++ b/src/main/java/com/milaboratory/mixcr/cli/ACommandWithSmartOverwrite.java @@ -1,16 +1,18 @@ package com.milaboratory.mixcr.cli; import com.milaboratory.mixcr.basictypes.ActionConfiguration; +import com.milaboratory.mixcr.basictypes.IOUtil; import com.milaboratory.mixcr.basictypes.PipelineConfiguration; import com.milaboratory.mixcr.basictypes.PipelineConfigurationReader; import picocli.CommandLine; +import picocli.CommandLine.Option; /** A command which allows resuming execution */ -public abstract class ACommandWithResume extends ACommandWithOutput { - @CommandLine.Option( - names = "--resume", - description = "Try to resume interrupted execution") - public boolean resume = false; +public abstract class ACommandWithSmartOverwrite extends ACommandWithOutput { + @Option(names = "--overwrite-if-required", + description = "Overwrite output file if it is corrupted or if it was generated from different input file " + + "or with different parameters. -f / --force-overwrite overrides this option.") + public boolean overwriteIfRequired = false; /** returns the unique run configuration */ public abstract ActionConfiguration getConfiguration(); @@ -22,6 +24,18 @@ public final String getOutput() { return getOutputFiles().get(0); } + private boolean outputFileInfoInitialized = false; + private IOUtil.MiXCRFileInfo outputFileInfo = null; + + public IOUtil.MiXCRFileInfo getOutputFileInfo() { + if (getOutputFiles().size() != 1) throw new RuntimeException(); + if (!outputFileInfoInitialized) { + outputFileInfo = IOUtil.getFileInfo(getOutput()); + outputFileInfoInitialized = true; + } + return outputFileInfo; + } + @Override public void validate() { super.validate(); @@ -32,18 +46,16 @@ public void validate() { /** whether to skip execution or not */ private boolean skipExecution = false; - protected boolean doOverwrite = true; - @Override public void handleExistenceOfOutputFile(String outFileName) { - if (doOverwrite && force) + if (forceOverwrite) // rewrite anyway return; // analysis supposed to be performed now PipelineConfiguration expectedPipeline = getFullPipelineConfiguration(); // history written in existing file - PipelineConfiguration actualPipeline = PipelineConfigurationReader.fromFileOrNull(outFileName); + PipelineConfiguration actualPipeline = PipelineConfigurationReader.fromFileOrNull(outFileName, getOutputFileInfo()); if (actualPipeline != null && expectedPipeline != null @@ -52,12 +64,12 @@ public void handleExistenceOfOutputFile(String outFileName) { String exists = "File " + outFileName + " already exists and contains correct " + "binary data obtained from the specified input file. "; - if (!resume) + if (!overwriteIfRequired) throwValidationException(exists + - "Use --resume option to skip execution (output file will remain unchanged) or use -f option " + - "to force overwrite it.", false); + "Use --overwrite-if-required option to skip execution (output file will remain unchanged) or " + + "use -f / --force-overwrite option to force overwrite it.", false); else { - warn("Skipping " + expectedPipeline.lastConfiguration().actionName() + " (--resume option specified). " + exists); + warn("Skipping " + expectedPipeline.lastConfiguration().actionName() + ". " + exists); // print warns in case different MiXCR versions for (int i = 0; i < expectedPipeline.pipelineSteps.length; i++) { @@ -65,15 +77,18 @@ public void handleExistenceOfOutputFile(String outFileName) { prev = actualPipeline.pipelineSteps[i].configuration, curr = expectedPipeline.pipelineSteps[i].configuration; if (!prev.versionId().equals(curr.versionId())) - warn(String.format("WARNING (--resume): %s was performed with outdated MiXCR version (%s). Consider re-running analysis using --force option.", + warn(String.format("WARNING (--overwrite-if-required): %s was performed with previous MiXCR version (%s). " + + "Consider re-running analysis using --force-overwrite option.", prev.actionName(), actualPipeline.pipelineSteps[i].versionOfMiXCR)); } - skipExecution = true; // nothing to do, just exit + skipExecution = true; // nothing to do in run0, just exit return; } } + if (overwriteIfRequired) + return; super.handleExistenceOfOutputFile(outFileName); } @@ -85,11 +100,4 @@ public final void run0() throws Exception { } public abstract void run1() throws Exception; -// -// public static abstract class ActionParametersWithResumeWithBinaryInput extends ActionParametersWithResumeOption { -// @Override -// public PipelineConfiguration getFullPipelineConfiguration() { -// return PipelineConfiguration.appendStep(PipelineConfigurationReader.fromFile(getInputFiles().get(0)), getInputFiles(), getConfiguration()); -// } -// } } diff --git a/src/main/java/com/milaboratory/mixcr/cli/ACommandWithResumeWithSingleInput.java b/src/main/java/com/milaboratory/mixcr/cli/ACommandWithSmartOverwriteWithSingleInput.java similarity index 57% rename from src/main/java/com/milaboratory/mixcr/cli/ACommandWithResumeWithSingleInput.java rename to src/main/java/com/milaboratory/mixcr/cli/ACommandWithSmartOverwriteWithSingleInput.java index 68b88f3e9..de2dbb339 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/ACommandWithResumeWithSingleInput.java +++ b/src/main/java/com/milaboratory/mixcr/cli/ACommandWithSmartOverwriteWithSingleInput.java @@ -1,5 +1,6 @@ package com.milaboratory.mixcr.cli; +import com.milaboratory.mixcr.basictypes.IOUtil; import com.milaboratory.mixcr.basictypes.PipelineConfiguration; import com.milaboratory.mixcr.basictypes.PipelineConfigurationReader; import picocli.CommandLine.Parameters; @@ -10,7 +11,7 @@ /** * */ -public abstract class ACommandWithResumeWithSingleInput extends ACommandWithResume { +public abstract class ACommandWithSmartOverwriteWithSingleInput extends ACommandWithSmartOverwrite { @Parameters(index = "0", description = "input file") public String in; @@ -27,8 +28,20 @@ public final List getInputFiles() { return Collections.singletonList(in); } + private boolean inputFileInfoInitialized = false; + private IOUtil.MiXCRFileInfo inputFileInfo = null; + + public IOUtil.MiXCRFileInfo getInputFileInfo() { + if (getInputFiles().size() != 1) throw new RuntimeException(); + if (!inputFileInfoInitialized) { + inputFileInfo = IOUtil.getFileInfo(in); + inputFileInfoInitialized = true; + } + return inputFileInfo; + } + @Override public PipelineConfiguration getFullPipelineConfiguration() { - return PipelineConfiguration.appendStep(PipelineConfigurationReader.fromFile(getInputFiles().get(0)), getInputFiles(), getConfiguration()); + return PipelineConfiguration.appendStep(PipelineConfigurationReader.fromFile(in, getInputFileInfo()), getInputFiles(), getConfiguration()); } } diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandAlign.java b/src/main/java/com/milaboratory/mixcr/cli/CommandAlign.java index 614ecb40f..7d7608f3a 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandAlign.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandAlign.java @@ -48,7 +48,7 @@ sortOptions = false, separator = " ", description = "Builds alignments with V,D,J and C genes for input sequencing reads.") -public class CommandAlign extends ACommandWithResume { +public class CommandAlign extends ACommandWithSmartOverwrite { static final String ALIGN_COMMAND_NAME = "align"; @Parameters(arity = "2..3", descriptionKey = "file", diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandAnalyze.java b/src/main/java/com/milaboratory/mixcr/cli/CommandAnalyze.java index cc6e242b3..bff05598a 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandAnalyze.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandAnalyze.java @@ -218,7 +218,7 @@ public void setChains(String chains) { if (chains.equalsIgnoreCase("xcr")) this.chains = Chains.ALL; else { - Chains c = Chains.parse(chains); + Chains c = Chains.parse(chains.toUpperCase()); if (c == null) throwValidationException("Illegal value " + chains + " for --receptor-type option."); this.chains = c; @@ -246,8 +246,9 @@ public void setChains(String chains) { @Option(names = {"-r", "--report"}, description = "Report file path") public String report = null; - @Option(names = {"--resume"}, description = "Try to resume aborted execution") - public boolean resume = false; +// @Option(names = {"--overwrite-if-required"}, description = "Overwrite output file if it is corrupted or if it was generated from different input file \" +\n" + +// " \"or with different parameters. -f / --force-overwrite overrides this option.") +// public boolean overwriteIfRequired = false; public String getReport() { if (report == null) @@ -257,12 +258,11 @@ public String getReport() { } private T inheritOptionsAndValidate(T parameters) { - if (resume && parameters instanceof ACommandWithResume) { - ((ACommandWithResume) parameters).resume = true; - ((ACommandWithResume) parameters).doOverwrite = false; - } - if (!resume && isForceOverwrite()) - parameters.force = true; + if (forceOverwrite) + parameters.forceOverwrite = true; + if (parameters instanceof ACommandWithSmartOverwrite) + ((ACommandWithSmartOverwrite) parameters).overwriteIfRequired = true; + parameters.quiet = true; parameters.validate(); parameters.quiet = false; @@ -481,10 +481,7 @@ public final CommandExport.CommandExportClones mkExport(String input, String out exportParameters.add("--filter-stops"); } - // logic with uber --resume and --force - if (resume || force) - exportParameters.add("-f"); - + exportParameters.add("--force-overwrite"); exportParameters.add("--chains"); exportParameters.add(chains.toString()); @@ -555,10 +552,7 @@ public String fNameForExportClones(String chains) { @Override public void handleExistenceOfOutputFile(String outFileName) { - if (!isForceOverwrite()) - throwValidationException("The destination file " + outFileName + - " already exists. Either remove it or use -f option to overwrite it (in this case you can also" + - " specify --resume option to prevent re-analyzing of intermediate files). "); + // Do nothing } @Override diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandAssemble.java b/src/main/java/com/milaboratory/mixcr/cli/CommandAssemble.java index 2d91ee83a..de31297be 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandAssemble.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandAssemble.java @@ -24,7 +24,7 @@ sortOptions = true, separator = " ", description = "Assemble clones.") -public class CommandAssemble extends ACommandWithResumeWithSingleInput { +public class CommandAssemble extends ACommandWithSmartOverwriteWithSingleInput { static final String ASSEMBLE_COMMAND_NAME = "assemble"; @Option(description = "Clone assembling parameters", diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandAssembleContigs.java b/src/main/java/com/milaboratory/mixcr/cli/CommandAssembleContigs.java index 4cf596ef1..c7154e14a 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandAssembleContigs.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandAssembleContigs.java @@ -30,7 +30,7 @@ sortOptions = true, separator = " ", description = "Assemble full sequences.") -public class CommandAssembleContigs extends ACommandWithResumeWithSingleInput { +public class CommandAssembleContigs extends ACommandWithSmartOverwriteWithSingleInput { static final String ASSEMBLE_CONTIGS_COMMAND_NAME = "assembleContigs"; public int threads = Runtime.getRuntime().availableProcessors(); diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandAssemblePartialAlignments.java b/src/main/java/com/milaboratory/mixcr/cli/CommandAssemblePartialAlignments.java index 6bf69323e..470dff1fa 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandAssemblePartialAlignments.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandAssemblePartialAlignments.java @@ -21,7 +21,7 @@ sortOptions = true, separator = " ", description = "Assembles partially aligned reads into longer sequences.") -public class CommandAssemblePartialAlignments extends ACommandWithResumeWithSingleInput { +public class CommandAssemblePartialAlignments extends ACommandWithSmartOverwriteWithSingleInput { static final String ASSEMBLE_PARTIAL_COMMAND_NAME = "assemblePartial"; @Option(names = "-O", description = "Overrides default parameter values.") diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandExport.java b/src/main/java/com/milaboratory/mixcr/cli/CommandExport.java index ddfafe3d9..6a3736408 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandExport.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandExport.java @@ -155,7 +155,7 @@ void run1(List> exporters) throws Excepti AutoCloseable reader = null; OutputPort source = null; - switch (IOUtil.detectFilType(in)) { + switch (IOUtil.getFileInfo(in).fileType) { case VDJCA: VDJCAlignmentsReader vdjcaReader = new VDJCAlignmentsReader(in, VDJCLibraryRegistry.getDefault()); reader = vdjcaReader; @@ -166,6 +166,8 @@ void run1(List> exporters) throws Excepti reader = clnaReader; source = clnaReader.readAllAlignments(); break; + case Clns: + throwExecutionException("Can't export alignments from *.clns file: " + in); default: throwExecutionException("Unknown file type: " + in); } diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandExtend.java b/src/main/java/com/milaboratory/mixcr/cli/CommandExtend.java index 9aa425212..d74814643 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandExtend.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandExtend.java @@ -29,7 +29,7 @@ sortOptions = true, separator = " ", description = "Impute alignments or clones with germline sequences.") -public class CommandExtend extends ACommandWithResumeWithSingleInput { +public class CommandExtend extends ACommandWithSmartOverwriteWithSingleInput { static final String EXTEND_COMMAND_NAME = "extend"; @Option(description = "Apply procedure only to alignments with specific immunological-receptor chains.", @@ -93,9 +93,7 @@ public ActionConfiguration getConfiguration() { @Override public void run1() throws Exception { - IOUtil.MiXCRFileType fileType = IOUtil.detectFilType(in); - - switch (fileType) { + switch (getInputFileInfo().fileType) { case VDJCA: processVDJCA(); break; diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandFilterAlignments.java b/src/main/java/com/milaboratory/mixcr/cli/CommandFilterAlignments.java index a8430520e..6c2bdde92 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandFilterAlignments.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandFilterAlignments.java @@ -30,7 +30,7 @@ sortOptions = true, separator = " ", description = "Filter alignments.") -public class CommandFilterAlignments extends ACommandWithResumeWithSingleInput { +public class CommandFilterAlignments extends ACommandWithSmartOverwriteWithSingleInput { static final String FILTER_ALIGNMENTS_COMMAND_NAME = "filterAlignments"; @Option(description = "Specifies immunological protein chain gene for an alignment. If many, " + diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandInfo.java b/src/main/java/com/milaboratory/mixcr/cli/CommandInfo.java index 0f604ff49..4192152c7 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandInfo.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandInfo.java @@ -1,6 +1,7 @@ package com.milaboratory.mixcr.cli; import cc.redberry.pipe.CUtils; +import com.milaboratory.mixcr.basictypes.IOUtil; import com.milaboratory.mixcr.basictypes.IOUtil.MiXCRFileType; import com.milaboratory.mixcr.basictypes.VDJCAlignments; import com.milaboratory.mixcr.basictypes.VDJCAlignmentsReader; @@ -16,8 +17,6 @@ import java.nio.file.Paths; import java.util.List; -import static com.milaboratory.mixcr.basictypes.IOUtil.detectFilType; - @Command(name = "info", sortOptions = true, hidden = true, @@ -35,8 +34,12 @@ public boolean isTableView() { return tableView; } + private IOUtil.MiXCRFileInfo info0 = null; + public MiXCRFileType getType() { - return detectFilType(input.get(0)); + if (info0 == null) + info0 = IOUtil.getFileInfo(input.get(0)); + return info0.fileType; } @Override @@ -44,7 +47,7 @@ public void validate() { super.validate(); MiXCRFileType type = getType(); for (String fileName : input) - if (detectFilType(fileName) != type) + if (IOUtil.getFileInfo(fileName).fileType != type) throwValidationException("Mixed file types: " + fileName); } @@ -56,7 +59,7 @@ public void run0() throws Exception { if (!tableView) throw new RuntimeException("Only table output is supported. Use -t option."); - switch (detectFilType(input.get(0))) { + switch (getType()) { case ClnA: case Clns: processClones(); diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandMergeAlignments.java b/src/main/java/com/milaboratory/mixcr/cli/CommandMergeAlignments.java index 69d1bc46e..02770edd3 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandMergeAlignments.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandMergeAlignments.java @@ -23,7 +23,7 @@ sortOptions = true, separator = " ", description = "Merge several *.vdjca files with alignments into a single alignments file.") -public class CommandMergeAlignments extends ACommandWithResume { +public class CommandMergeAlignments extends ACommandWithSmartOverwrite { static final String MERGE_ALIGNMENTS_COMMAND_NAME = "mergeAlignments"; @Parameters(description = "[input_file1.vdjca [input_file2.vdjca ....]] output_file.vdjca", arity = "2..*") diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandSlice.java b/src/main/java/com/milaboratory/mixcr/cli/CommandSlice.java index 6dc6e3f7e..08acdf565 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandSlice.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandSlice.java @@ -24,7 +24,7 @@ sortOptions = true, separator = " ", description = "Slice ClnA file.") -public class CommandSlice extends ACommandWithResumeWithSingleInput { +public class CommandSlice extends ACommandWithSmartOverwriteWithSingleInput { static final String SLICE_COMMAND_NAME = "slice"; @Option(description = "List of read (for .vdjca) / clone (for .clns/.clna) ids to export.", @@ -40,9 +40,7 @@ public ActionConfiguration getConfiguration() { public void run1() throws Exception { Collections.sort(ids); - IOUtil.MiXCRFileType fileType = IOUtil.detectFilType(in); - - switch (fileType) { + switch (getInputFileInfo().fileType) { case VDJCA: sliceVDJCA(); break; diff --git a/src/main/java/com/milaboratory/mixcr/cli/CommandSortAlignments.java b/src/main/java/com/milaboratory/mixcr/cli/CommandSortAlignments.java index 2b7137a68..99b9506b1 100644 --- a/src/main/java/com/milaboratory/mixcr/cli/CommandSortAlignments.java +++ b/src/main/java/com/milaboratory/mixcr/cli/CommandSortAlignments.java @@ -29,7 +29,7 @@ sortOptions = true, separator = " ", description = "Sort alignments in vdjca file by read id.") -public class CommandSortAlignments extends ACommandWithResumeWithSingleInput { +public class CommandSortAlignments extends ACommandWithSmartOverwriteWithSingleInput { static final String SORT_ALIGNMENTS_COMMAND_NAME = "sortAlignments"; @Override