From 25e89f0a8eb95ed4471c6d1c5b641f74d1f07ad1 Mon Sep 17 00:00:00 2001 From: Jorge Orpinel Date: Tue, 28 May 2019 23:02:26 -0500 Subject: [PATCH 1/4] dvc: fixes dvc.scm.Git.get_diff_trees docstring imprecision. Parentheses were used in the "Returns:" description but its a dict, so changed to curly brackets; and specified the 2 possible forms of the return structure. --- dvc/scm/git/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 3522f74d00..690b54c3c1 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -297,8 +297,8 @@ def get_diff_trees(self, a_ref, b_ref=None): b_ref (str): optional second git reference, default None Returns: - dict: dictionary with keys: (a_tree, b_tree, a_ref, b_ref, - equal) + dict: dictionary with keys: {a_ref, b_ref, equal} + or {a_ref, b_ref, a_tree, b_tree} """ diff_dct = {DIFF_EQUAL: False} trees, commits = self._get_diff_trees(a_ref, b_ref) From d66b4513fe60ecfd21cee4cfdb4d96c9e06a1a1d Mon Sep 17 00:00:00 2001 From: Jorge Orpinel Date: Tue, 28 May 2019 23:03:43 -0500 Subject: [PATCH 2/4] dvc: updates zsh autocomplete script with current command help messages. Comparing vs actually running each command's `-h` help output. --- dvc/command/remove.py | 2 +- scripts/completion/dvc.zsh | 145 +++++++++++++++++++------------------ 2 files changed, 77 insertions(+), 70 deletions(-) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index aa455702bb..6f27066742 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -57,7 +57,7 @@ def add_parser(subparsers, parent_parser): "--outs", action="store_true", default=True, - help="Only remove DVC file outputs.(default)", + help="Only remove DVC file outputs. (Default)", ) remove_parser_group.add_argument( "-p", diff --git a/scripts/completion/dvc.zsh b/scripts/completion/dvc.zsh index 4ceef21b97..a1a0849d58 100644 --- a/scripts/completion/dvc.zsh +++ b/scripts/completion/dvc.zsh @@ -44,21 +44,21 @@ _dvc_commands() { _describe 'dvc commands' _commands } -_dvc_global_options=( - "(-)"{-h,--help}"[Show help message related to the command]" - "(-)"{-q,--quiet}"[Be quiet.]" - "(-)"{-V,--verbose}"[Be verbose.]" +_dvc_options=( + "(-)"{-h,--help}"[Show help message.]" + "(-)"{-V,--version}"[Show program's version]" ) -_dvc_options=( - "(-)"{-h,--help}"[Show this help message and exit]" - "(-)"{-v,--version}"[Show program's version]" +_dvc_global_options=( + "(-)"{-h,--help}"[Show help message related to the command.]" + "(-)"{-q,--quiet}"[Be quiet.]" + "(-)"{-v,--verbose}"[Be verbose.]" ) _dvc_add=( {-R,--recursive}"[Recursively add each file under the directory.]" - {-f,--file}"[Specify name of the stage file.]:File:_files" "--no-commit[Don't put files/directories into cache.]" + {-f,--file}"[Specify name of the DVC file it generates.]:File:_files" "1:File:_files" ) @@ -67,27 +67,28 @@ _dvc_cache=( ) _dvc_checkout=( - {-f,--force}"[Do not prompt when removing working directory files.]" {-d,--with-deps}"[Checkout all dependencies of the specified target.]" + {-f,--force}"[Do not prompt when removing working directory files.]" + {-R,--recursive}"[Checkout all subdirectories of the specified directory.]" "1:Stages:_files -g '(*.dvc|Dvcfile)'" ) _dvc_commit=( "*:Stages:_files -g '(*.dvc|Dvcfile)'" - {-d,--with-deps}"[Commit all dependencies of the specified target.]" {-f,--force}"[Commit even if checksums for dependencies/outputs changed.]" + {-d,--with-deps}"[Commit all dependencies of the specified target.]" {-R,--recursive}"[Commit cache for subdirectories of the specified directory.]" ) _dvc_config=( - {-u,--unset}"[Unset option.]" - "--local[Unset option.]" - "--system[Use system config.]" "--global[Use global config.]" + "--system[Use system config.]" + "--local[Use local config.]" + {-u,--unset}"[Unset option.]" ) _dvc_destroy=( - {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]" + {-f,--force}"[Force destruction.]" ) _dvc_diff=( @@ -95,35 +96,36 @@ _dvc_diff=( ) _dvc_fetch=( + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-r,--remote}"[Remote repository to fetch from.]:Remote repository:" + {-a,--all-branches}"[Fetch cache for all branches.]" + {-T,--all-tags}"[Fetch cache for all tags.]" + {-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]" + {-R,--recursive}"[Fetch cache for subdirectories of specified directory.]" "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" - {-R,--recursive}"[Fetch cache for subdirectories of the specified directory.]" ) _dvc_gc=( - {-a,--all-branches}"[Collect garbage for all branches]" - {-T,--all-tags}"[Collect garbage for all tags]" - {-c,--cloud}"[Collect garbage in remote repository]" - {-r,--remote}"[Remote repository to collect garbage in]:Remote repository:" - {-f,--force}"[Force garbage collection - automatically agree to all prompts.]" - - {-p,--projects}"[Keep data files required by these projects in addition to the current one. Useful if you share a single cache across repos.]:Repos:_files" + {-a,--all-branches}"[Keep data files for the tips of all git branches.]" + {-T,--all-tags}"[Keep data files for all git tags.]" + {-c,--cloud}"[Collect garbage in remote repository.]" + {-r,--remote}"[Remote storage to collect garbage in.]:Remote repository:" + {-f,--force}"[Force garbage collection - automatically agree to all prompts.]:Repos:_files" + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + {-p,--projects}"[Keep data files required by these projects in addition to the current one.]:Repos:_files" ) _dvc_import=( "--resume[Resume previously started download.]" + {-f,--file}"[Specify name of the DVC file it generates.]:File:_files" "1:URL:" "2:Output:" ) _dvc_init=( - "--no-scm[Initiate dvc in directory that is not tracket by any scm tool]" - {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]" + "--no-scm[Initiate dvc in directory that is not tracked by any scm tool.]" + {-f,--force}"[Overwrite existing '.dvc' directory. This operation removes local cache.]" ) _dvc_install=() @@ -146,26 +148,26 @@ _dvc_pipeline=( ) _dvc_pull=( - "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-r,--remote}"[Remote repository to pull from.]:Remote repository:" + {-a,--all-branches}"[Fetch cache for all branches.]" + {-T,--all-tags}"[Fetch cache for all tags.]" + {-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]" {-f,--force}"[Do not prompt when removing working directory files.]" {-R,--recursive}"[Pull cache for subdirectories of the specified directory.]" + "*:Stages:_files -g '(*.dvc|Dvcfile)'" ) _dvc_push=( + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-r,--remote}"[Remote repository to push to.]:Remote repository:" + {-a,--all-branches}"[Push cache for all branches.]" + {-T,--all-tags}"[Push cache for all tags.]" + {-d,--with-deps}"[Push cache for all dependencies of the specified target.]" + {-R,--recursive}"[Push cache from subdirectories of specified directory.]" "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" - {-R,--recursive}"[Push cache for subdirectories of the specified directory.]" ) _dvc_remote=( @@ -175,53 +177,58 @@ _dvc_remote=( _dvc_remove=( "*:Stages:_files -g '(*.dvc|Dvcfile)'" "--dry[Only print the commands that would be executed without actually executing]" - {-o,--outs}"[Only remove DVC file outputs.(default)]" - {-p,--purge}"[Remove DVC file and all its outputs]" + {-o,--outs}"[Only remove DVC file outputs. (Default)]" + {-p,--purge}"[Remove DVC file and all its outputs.]" {-f,--force}"[Force purge.]" ) _dvc_repro=( - "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--dry[Only print the commands that would be executed without actually executing]" - "--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]" - "--no-commit[Don't put files/directories into cache.]" {-f,--force}"[Reproduce even if dependencies were not changed.]" {-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]" + {-c,--cwd}"[Directory within your repo to reproduce from.]:CWD:_files -/" {-m,--metrics}"[Show metrics after reproduction.]" + "--dry[Only print the commands that would be executed without actually executing]" {-i,--interactive}"[Ask for confirmation before reproducing each stage.]" {-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]" - {-P,--all-pipelines}"[Reproduce all pipelines in the project.]" - {-d,--downstream}"[Reproduce the pipeline starting from the specified stage.]" + {-P,--all-pipelines}"[Reproduce all pipelines in the repo.]" + {-R,--recursive}"[Reproduce all stages in the specified directory.]" + "--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]" + "--no-commit[Don't put files/directories into cache.]" + "--downstream[Reproduce the pipeline starting from the specified stage.]" + "*:Stages:_files -g '(*.dvc|Dvcfile)'" ) _dvc_root=() _dvc_run=( + "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files" + "*"{-o,--outs}"[Declare output file or directory.]:Output data:_files" + "*"{-O,--outs-no-cache}"[Declare output file or directory (do not put into DVC cache).]:Output regular:_files" + "*"{-m,--metrics}"[Declare output metric file or directory.]:Metrics:_files" + "*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache).]:Metrics (no cache):_files" + {-f,--file}"[Specify name of the DVC file it generates.]:File:_files" + {-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/" + {-w,--wdir}"[Directory within your repo to run your command in.]:WDIR:_files -/" "--no-exec[Only create stage file without actually running it.]" + {-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]" "--overwrite-dvcfile[Overwrite existing dvc file without asking for confirmation.]" "--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]" + "--remove-outs[Deprecated, this is now the default behavior]" "--no-commit[Don't put files/directories into cache.]" - {-f,--file}"[Specify name of the stage file.]:File:_files" - {-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/" - {-w,--wdir}"[Directory to run your command and place state file in.]:WDIR:_files -/" - "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files" - "*"{-o,--outs}"[Declare output data file or data directory.]:Output data:_files" - "*"{-O,--outs-no-cache}"[Declare output regular file or directory.]:Output regular:_files" - "*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache)]:Metrics (no cache):_files" - "*"{-m,--metrics}"[Declare output metric file or directory]:Metrics:_files" - {-y,--yes}"[Automatic 'yes' answer to all prompts.]" + "--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files" + "--outs-persist-no-cache[Declare output file or directory that will not be removed upon repro (do not put into DVC cache).]:Output persistent regular:_files" ) _dvc_status=( + {-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:" + "--show-checksums[Show checksums instead of file names.]" + {-q,--quiet}"[Suppresses all output. Exit with 0 if pipeline is up to date, otherwise 1.]" + {-c,--cloud}"[Show status of a local cache compared to a remote repository.]" + {-r,--remote}"[Remote repository to compare local cache to.]:Remote repository:" + {-a,--all-branches}"[Show status of a local cache compared to a remote repository for all branches.]" + {-T,--all-tags}"[Show status of a local cache compared to a remote repository for all tags.]" + {-d,--with-deps}"[Show status for all dependencies of the specified target.]" "*:Stages:_files -g '(*.dvc|Dvcfile)'" - "--show-checksums[Show checksums instead of file names]" - {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:" - {-r,--remote}"[Remote repository to pull from]:Remote repository:" - {-a,--all-branches}"[Fetch cache for all branches]" - {-T,--all-tags}"[Fetch cache for all tags]" - {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]" - {-c,--cloud}"[Show status of a local cache compared to a remote repository]" - {-q,--quiet}"[Suppresses all output. Exit with 0 if pipeline is up]" ) _dvc_unlock=( From 9dde091087c4729b21a19deaf9bb7afe4ae52aa3 Mon Sep 17 00:00:00 2001 From: Jorge Orpinel Date: Tue, 28 May 2019 23:25:58 -0500 Subject: [PATCH 3/4] dvc: begins updating bask autocomplete script... Only reorders existing commands and adds comments for those missing. --- scripts/completion/dvc.bash | 40 +++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/scripts/completion/dvc.bash b/scripts/completion/dvc.bash index d311494594..28d306ada8 100644 --- a/scripts/completion/dvc.bash +++ b/scripts/completion/dvc.bash @@ -12,30 +12,36 @@ _dvc_commands='init destroy add import checkout run pull push fetch \ status repro remove move unprotect gc config remote metrics \ install root lock unlock pipeline commit' -_dvc_global_options="-h --help -q --quiet -V --verbose" _dvc_options="-h --help -v --version" -_dvc_init="--no-scm -f --force" -_dvc_destroy="-f --force" +_dvc_global_options="-h --help -q --quiet -V --verbose" + _dvc_add="-R --recursive -f --file --no-commit" -_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes --overwrite-dvc-file --ignore-build-cache --remove-outs --no-commit -w --wdir" -_dvc_pull="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -f --force -R --recursive" -_dvc_push="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive" +#_dvc_cache=? +_dvc_checkout="$(compgen -G '*.dvc')" +#_dvc_commit=? +_dvc_config="-u --unset --local --system --global" +_dvc_destroy="-f --force" +#_dvc_diff=? _dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive" -_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud -q --quiet" -_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -s --single-item" -_dvc_remove="--dry -o --outs -p --purge --force" _dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote -f --force -p --project" -_dvc_config="-u --unset --local --system --global" -_dvc_checkout="$(compgen -G '*.dvc')" -_dvc_move="$(compgen -G '*')" -_dvc_lock="$(compgen -G '*.dvc')" -_dvc_unlock="$(compgen -G '*.dvc')" _dvc_import="--resume" -_dvc_remote="" -_dvc_metrics="" +_dvc_init="--no-scm -f --force" _dvc_install="" -_dvc_root="" +_dvc_lock="$(compgen -G '*.dvc')" +_dvc_metrics="" +_dvc_move="$(compgen -G '*')" _dvc_pipeline="" +_dvc_pull="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -f --force -R --recursive" +_dvc_push="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive" +_dvc_remote="" +_dvc_remove="--dry -o --outs -p --purge --force" +_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -s --single-item" +_dvc_root="" +_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes --overwrite-dvc-file --ignore-build-cache --remove-outs --no-commit -w --wdir" +_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud -q --quiet" +_dvc_unlock="$(compgen -G '*.dvc')" +#_dvc_unprotect=? +#_dvc_version=? # Notes: # From bcc92ab4b51c1579e7956595f982ea8c454d5b4b Mon Sep 17 00:00:00 2001 From: Jorge Orpinel Date: Wed, 29 May 2019 20:10:18 -0500 Subject: [PATCH 4/4] scripts: removes temporary comments in bash autocomplete script. Per https://github.com/iterative/dvc/pull/2064#pullrequestreview-243578115 Related to #2069 --- scripts/completion/dvc.bash | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/completion/dvc.bash b/scripts/completion/dvc.bash index 28d306ada8..adc389e0f4 100644 --- a/scripts/completion/dvc.bash +++ b/scripts/completion/dvc.bash @@ -16,12 +16,9 @@ _dvc_options="-h --help -v --version" _dvc_global_options="-h --help -q --quiet -V --verbose" _dvc_add="-R --recursive -f --file --no-commit" -#_dvc_cache=? _dvc_checkout="$(compgen -G '*.dvc')" -#_dvc_commit=? _dvc_config="-u --unset --local --system --global" _dvc_destroy="-f --force" -#_dvc_diff=? _dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive" _dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote -f --force -p --project" _dvc_import="--resume" @@ -40,8 +37,6 @@ _dvc_root="" _dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes --overwrite-dvc-file --ignore-build-cache --remove-outs --no-commit -w --wdir" _dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud -q --quiet" _dvc_unlock="$(compgen -G '*.dvc')" -#_dvc_unprotect=? -#_dvc_version=? # Notes: #