Skip to content

Commit

Permalink
Stats fixes/updates
Browse files Browse the repository at this point in the history 
linting

linting

Fixing tests

Changing to argv from args

lint

test_parse_dag

test_parse_dag

Review fixes

Added field for missing UIDs

Additional function to get home dir
  • Loading branch information
@idomic
idomic committed Jan 20, 2022
1 parent c19cf84 commit a5bcdc9
Show file tree
Hide file tree
Showing 14 changed files with 479 additions and 101 deletions.
1 change: 1 addition & 0 deletions setup.py
View file
Expand Up @@ -153,6 +153,7 @@ def read(name):
'humanize',
'tqdm',
'posthog',
'distro',
'importlib_resources;python_version<"3.7"',
# for code normalization, parso is also needed for inferring upstream
# dependencies in jupyter notebooks
Expand Down
13 changes: 11 additions & 2 deletions src/ploomber/cli/build.py
View file
Expand Up @@ -40,7 +40,14 @@ def main(render_only=False):
# users may try to run "ploomber build {name}" to build a single task
if len(sys.argv) > 1 and not sys.argv[1].startswith('-'):
suggestion = 'ploomber task {task-name}'
parser.error(f'{parser.prog!r} does not take positional arguments.\n'
cmd_name = parser.prog
telemetry.log_api("unsupported_build_cmd",
metadata={
'cmd_name': cmd_name,
'suggestion': suggestion,
'argv': sys.argv
})
parser.error(f'{cmd_name!r} does not take positional arguments.\n'
f'To build a single task, try: {suggestion!r}')

dag, args = parser.load_from_entry_point_arg()
Expand All @@ -65,5 +72,7 @@ def main(render_only=False):
print(report)
end_time = datetime.datetime.now()
telemetry.log_api("ploomber_build",
total_runtime=str(end_time - start_time))
total_runtime=str(end_time - start_time),
dag=dag,
metadata={'argv': sys.argv})
return dag
60 changes: 56 additions & 4 deletions src/ploomber/cli/cli.py
View file
Expand Up @@ -48,13 +48,34 @@ def scaffold(conda, package, entry_point, empty):
template = '-e/--entry-point is not compatible with the {flag} flag'

if entry_point and conda:
raise click.ClickException(template.format(flag='--conda'))
err = template.format(flag='--conda')
telemetry.log_api("scaffold_error",
metadata={
'type': 'entry_and_conda_flag',
'exception': err,
'argv': sys.argv
})
raise click.ClickException(err)

if entry_point and package:
raise click.ClickException(template.format(flag='--package'))
err = template.format(flag='--package')
telemetry.log_api("scaffold_error",
metadata={
'type': 'entry_and_package_flag',
'exception': err,
'argv': sys.argv
})
raise click.ClickException(err)

if entry_point and empty:
raise click.ClickException(template.format(flag='--empty'))
err = template.format(flag='--empty')
telemetry.log_api("scaffold_error",
metadata={
'type': 'entry_and_empty_flag',
'exception': err,
'argv': sys.argv
})
raise click.ClickException(err)

# try to load a dag by looking in default places
if entry_point is None:
Expand All @@ -67,14 +88,31 @@ def scaffold(conda, package, entry_point, empty):
Path(entry_point),
)
except Exception as e:
telemetry.log_api("scaffold_error",
metadata={
'type': 'dag_load_failed',
'exception': e,
'argv': sys.argv
})
raise click.ClickException(e) from e

if loaded:
# existing pipeline, add tasks
spec, _, path_to_spec = loaded
_scaffold.add(spec, path_to_spec)
telemetry.log_api("ploomber_scaffold",
dag=loaded,
metadata={
'type': 'add_task',
'argv': sys.argv
})
else:
# no pipeline, create base project
telemetry.log_api("ploomber_scaffold",
metadata={
'type': 'base_project',
'argv': sys.argv
})
scaffold_project.cli(project_path=None,
conda=conda,
package=package,
Expand Down Expand Up @@ -109,6 +147,12 @@ def examples(name, force, branch, output):
except click.ClickException:
raise
except Exception as e:
telemetry.log_api("examples_error",
metadata={
'type': 'runtime_error',
'exception': e,
'argv': sys.argv
})
raise RuntimeError(
'An error happened when executing the examples command. Check out '
'the full error message for details. Downloading the examples '
Expand Down Expand Up @@ -150,11 +194,19 @@ def cmd_router():
fn()
elif cmd_name in alias:
suggestion = alias[cmd_name]
telemetry.log_api("unsupported_build_cmd",
metadata={
'cmd_name': cmd_name,
'suggestion': suggestion,
'argv': sys.argv
})
_exit_with_error_message("Try 'ploomber --help' for help.\n\n"
f"Error: {cmd_name!r} is not a valid command."
f" Did you mean {suggestion!r}?")
else:
telemetry.log_api("unsupported-api-call", metadata={'argv': sys.argv})
if cmd_name not in ['examples', 'scaffold']:
telemetry.log_api("unsupported-api-call",
metadata={'argv': sys.argv})
cli()


Expand Down
17 changes: 17 additions & 0 deletions src/ploomber/cli/examples.py
View file
Expand Up @@ -2,6 +2,7 @@
import stat
import os
import csv
import sys
from datetime import datetime
import json
import shutil
Expand All @@ -13,6 +14,7 @@

from ploomber.io.terminalwriter import TerminalWriter
from ploomber.table import Table
from ploomber.telemetry import telemetry

from pygments.formatters.terminal import TerminalFormatter
from pygments.lexers.markup import MarkdownLexer
Expand Down Expand Up @@ -250,6 +252,11 @@ def main(name, force=False, branch=None, output=None):
selected = manager.path_to(name)

if not selected.exists():
telemetry.log_api("examples_error",
metadata={
'type': 'wrong_example_name',
'example_name': name
})
click.echo(f'There is no example named {name!r}.\n'
'To list examples: ploomber examples\n'
'To update local copy: ploomber examples -f')
Expand All @@ -259,6 +266,11 @@ def main(name, force=False, branch=None, output=None):
tw.sep('=', f'Copying example {name!r} to {output}/', green=True)

if Path(output).exists():
telemetry.log_api("examples_error",
metadata={
'type': 'duplicated_output',
'output_name': output
})
raise click.ClickException(
f"{output!r} already exists in the current working "
"directory, please rename it or move it "
Expand All @@ -279,3 +291,8 @@ def main(name, force=False, branch=None, output=None):
f'\n* conda env create -f environment.yml'
f'\n* pip install -r requirements.txt\n')
tw.sep('=', blue=True)
telemetry.log_api("ploomber_examples",
metadata={
'argv': sys.argv,
'example_name': name
})
79 changes: 56 additions & 23 deletions src/ploomber/cli/install.py
View file
Expand Up @@ -45,29 +45,41 @@ def main(use_lock):
err = ("Expected and environment.lock.yaml "
"(conda) or requirements.lock.txt (pip) in the current "
"directory. Add one of them and try again.")
telemetry.log_api("install-exception-lock",
metadata={'Exception': err})
telemetry.log_api("install-error",
metadata={
'type': 'no_lock',
'exception': err
})
raise exceptions.ClickException(err)
elif not use_lock and not HAS_ENV_YML and not HAS_REQS_TXT:
err = ("Expected an environment.yaml (conda)"
" or requirements.txt (pip) in the current directory."
" Add one of them and try again.")
telemetry.log_api("install-exception-requirements",
metadata={'Exception': err})
telemetry.log_api("install-error",
metadata={
'type': 'no_env_requirements',
'exception': err
})
raise exceptions.ClickException(err)
elif (not HAS_CONDA and use_lock and HAS_ENV_LOCK_YML
and not HAS_REQS_LOCK_TXT):
err = ("Found env environment.lock.yaml "
"but conda is not installed. Install conda or add a "
"requirements.lock.txt to use pip instead")
telemetry.log_api("install-exception-conda",
metadata={'Exception': err})
telemetry.log_api("install-error",
metadata={
'type': 'no_conda',
'exception': err
})
raise exceptions.ClickException(err)
elif not HAS_CONDA and not use_lock and HAS_ENV_YML and not HAS_REQS_TXT:
err = ("Found environment.yaml but conda is not installed."
" Install conda or add a requirements.txt to use pip instead")
telemetry.log_api("install-exception-conda2",
metadata={'Exception': err})
telemetry.log_api("install-error",
metadata={
'type': 'no_conda2',
'exception': err
})
raise exceptions.ClickException(err)
elif HAS_CONDA and use_lock and HAS_ENV_LOCK_YML:
main_conda(start_time, use_lock=True)
Expand Down Expand Up @@ -154,10 +166,16 @@ def main_conda(start_time, use_lock):
current_env = Path(shutil.which('python')).parents[1].name

if env_name == current_env:
raise RuntimeError(f'{env_yml} will create an environment '
f'named {env_name!r}, which is the current active '
'environment. Move to a different one and try '
'again (e.g., "conda activate base")')
err = (f'{env_yml} will create an environment '
f'named {env_name!r}, which is the current active '
'environment. Move to a different one and try '
'again (e.g., "conda activate base")')
telemetry.log_api("install-error",
metadata={
'type': 'env_running_conflict',
'exception': err
})
raise RuntimeError(err)

# get current installed envs
conda = shutil.which('conda')
Expand All @@ -175,9 +193,15 @@ def main_conda(start_time, use_lock):
])

if already_installed:
raise ValueError(f'Environment {env_name!r} already exists, '
f'delete it and try again '
f'(conda env remove --name {env_name})')
err = (f'Environment {env_name!r} already exists, '
f'delete it and try again '
f'(conda env remove --name {env_name})')
telemetry.log_api("install-error",
metadata={
'type': 'duplicate_env',
'exception': err
})
raise ValueError(err)

pkg_manager = mamba if mamba else conda
cmdr.run(pkg_manager,
Expand Down Expand Up @@ -225,11 +249,15 @@ def _find_conda_root(conda_bin):
# on linux miniconda3, anaconda and miniconda
if parent.name.lower() in {'miniconda3', 'miniconda', 'anaconda3'}:
return parent

raise RuntimeError(
'Failed to locate conda root from '
f'directory: {str(conda_bin)!r}. Please submit an issue: '
'https://github.com/ploomber/ploomber/issues/new')
err = ('Failed to locate conda root from '
f'directory: {str(conda_bin)!r}. Please submit an issue: '
'https://github.com/ploomber/ploomber/issues/new')
telemetry.log_api("install-error",
metadata={
'type': 'no_conda_root',
'exception': err
})
raise RuntimeError(err)


def _path_to_pip_in_env_with_name(conda_bin, env_name):
Expand All @@ -246,9 +274,14 @@ def _locate_pip_inside_conda(env_name):

# this might happen if the environment does not contain python/pip
if not Path(pip).exists():
raise FileNotFoundError(
f'Could not locate pip in environment {env_name!r}, make sure '
'it is included in your environment.yml and try again')
err = (f'Could not locate pip in environment {env_name!r}, make sure '
'it is included in your environment.yml and try again')
telemetry.log_api("install-error",
metadata={
'type': 'no_pip_env',
'exception': err
})
raise FileNotFoundError(err)

return pip

Expand Down
14 changes: 11 additions & 3 deletions src/ploomber/cli/interact.py
View file
Expand Up @@ -20,12 +20,20 @@ def main():
try:
dag.render()
except Exception:
print('Your dag failed to render, but you can still inspect the '
'object to debug it.\n')
err = ('Your dag failed to render, but you can still inspect the '
'object to debug it.\n')
telemetry.log_api("interact_error",
dag=dag,
metadata={
'type': 'dag_render_failed',
'exception': err
})
print(err)

end_time = datetime.datetime.now()
telemetry.log_api("ploomber_interact",
total_runtime=str(end_time - start_time))
total_runtime=str(end_time - start_time),
dag=dag)

# NOTE: do not use embed here, we must use start_ipython, see here:
# https://github.com/ipython/ipython/issues/8918
Expand Down

0 comments on commit a5bcdc9

Please sign in to comment.