Skip to content
Permalink
Browse files

Doc nits and updates (#1224)

  • Loading branch information...
mgasner committed Apr 12, 2019
1 parent c9ae931 commit be5a9cb74471d780e19f7132338dea2a8bd64c81
Showing with 1,949 additions and 1,223 deletions.
  1. +1 −4 .circleci/config.yml
  2. +0 −124 CHANGES.rst
  3. +1 −3 Makefile
  4. +2 −0 pull_request_template.md
  5. +0 −2 pyproject.toml
  6. +1 −2 python_modules/Makefile
  7. +12 −2 python_modules/dagster/dagster/__init__.py
  8. +1 −6 python_modules/dagster/dagster/tutorials/intro_tutorial/actual_dag.py
  9. +3 −8 python_modules/dagster/dagster/tutorials/intro_tutorial/config.py
  10. +5 −19 python_modules/dagster/dagster/tutorials/intro_tutorial/configuration_schemas.py
  11. +2 −4 python_modules/dagster/dagster/tutorials/intro_tutorial/execution_context.py
  12. +2 −6 python_modules/dagster/dagster/tutorials/intro_tutorial/expectations.py
  13. +2 −9 python_modules/dagster/dagster/tutorials/intro_tutorial/hello_dag.py
  14. +1 −3 python_modules/dagster/dagster/tutorials/intro_tutorial/hello_world.py
  15. +2 −7 python_modules/dagster/dagster/tutorials/intro_tutorial/inputs.py
  16. +6 −20 python_modules/dagster/dagster/tutorials/intro_tutorial/multiple_outputs.py
  17. +3 −11 python_modules/dagster/dagster/tutorials/intro_tutorial/pipeline_cli_execution.py
  18. +0 −2 python_modules/dagster/dagster/tutorials/intro_tutorial/pyproject.toml
  19. +8 −40 python_modules/dagster/dagster/tutorials/intro_tutorial/resources.py
  20. +1 −3 python_modules/dagster/dagster/tutorials/intro_tutorial/unittesting.py
  21. +1 −11 python_modules/dagster/dagster/tutorials/utils.py
  22. +12 −2 python_modules/dagster/docs/conf.py
  23. +12 −0 python_modules/dagster/docs/sections/api/apidocs/execution.rst
  24. +80 −0 python_modules/dagster/docs/sections/community/code_of_conduct.rst
  25. +1 −0 python_modules/dagster/docs/sections/community/community.rst
  26. +228 −0 python_modules/dagster/docs/sections/community/release_notes.rst
  27. +39 −16 python_modules/dagster/docs/sections/learn/guides/028_to_030_new_concepts.md
  28. +71 −54 python_modules/dagster/docs/sections/learn/guides/028_to_030_upgrade_guide.md
  29. +2 −1 python_modules/dagster/docs/sections/learn/principles.rst
  30. +2 −2 python_modules/dagster/docs/sections/learn/tutorial/config.rst
  31. +5 −8 python_modules/dagster/docs/sections/learn/tutorial/configuration_schemas.rst
  32. +1 −1 python_modules/dagster/docs/sections/learn/tutorial/execution_context.rst
  33. +1 −1 python_modules/dagster/docs/sections/learn/tutorial/expectations.rst
  34. +2 −2 python_modules/dagster/docs/sections/learn/tutorial/hello_world.rst
  35. +1 −1 python_modules/dagster/docs/sections/learn/tutorial/inputs.rst
  36. +2 −2 python_modules/dagster/docs/sections/learn/tutorial/multiple_outputs.rst
  37. +8 −7 python_modules/dagster/docs/sections/learn/tutorial/resources.rst
  38. +2 −2 python_modules/dagster/docs/sections/learn/tutorial/unittesting.rst
  39. +1,404 −837 python_modules/dagster/docs/snapshots/snap_test_doc_build.py
  40. +22 −1 python_modules/dagster/docs/test_doc_build.py
@@ -227,11 +227,8 @@ jobs:

- run:
name: Format
command: black python_modules --line-length 100 -S --fast --check --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|python_modules/dagster/dagster/tutorials/|snapshots/|__scaffold\.py" -N
command: black python_modules --line-length 100 -S --fast --check --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|python_modules/dagster/dagster/|snapshots/" -N

- run:
name: Format
command: black python_modules/dagster/dagster/tutorials examples --line-length 79 -S --fast --check -N

lint:
docker:

This file was deleted.

Oops, something went wrong.
@@ -12,6 +12,4 @@ update_doc_snapshot:
pytest python_modules/dagster/docs --snapshot-update

black:
black python_modules --line-length 100 -S --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|python_modules/dagster/dagster/tutorials/|snapshots/|__scaffold\.py" -N
black python_modules/dagster/dagster/tutorials examples --line-length 79 -S --fast -N

black python_modules --line-length 100 -S --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/" -N
@@ -0,0 +1,2 @@
**STOP!** Did you remember to make changes to the release notes reflecting any changes to public
APIs or new user-facing functionality introduced in this diff?
@@ -13,8 +13,6 @@ exclude = '''
| \.nox
| \.tox
| \.venv
| python_modules/dagster/dagster/tutorials
| snapshots
| python_modules/dagster-airflow/dagster_airflow_tests/test_project
)/
'''
@@ -8,8 +8,7 @@ watch:
--drop dagster/dagster dagster/dagster_tests/ dagster-ge/dagster_ge dagster-ge/dagster_ge_tests dagit/dagit dagit/dagit_tests

black:
black . --line-length 100 -S --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|python_modules/dagster/dagster/tutorials/|snapshots/|__scaffold\.py" -N
black ./dagster/dagster/tutorials ../examples --line-length 79 -S --fast -N
black . --line-length 100 -S --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/" -N


pylint:
@@ -83,42 +83,50 @@

from .version import __version__


__all__ = [
# Definition
'DependencyDefinition',
'ExpectationDefinition',
'ExpectationResult',
'Field',
'InputDefinition',
'Materialization',
'OutputDefinition',
'PipelineContextDefinition',
'PipelineDefinition',
'RepositoryDefinition',
'ResourceDefinition',
'resource',
'Result',
'SolidDefinition',
'SolidInstance',
'Materialization',
# Decorators
'lambda_solid',
'resource',
'solid',
'MultipleResults',
# Execution
'execute_pipeline_iterator',
'execute_pipeline',
'ExecutionContext',
'InProcessExecutorConfig',
'MultiprocessExecutorConfig',
'PipelineConfigEvaluationError',
'PipelineExecutionResult',
'RunConfig',
'RunStorageMode',
'SolidExecutionResult',
# Errors
'DagsterEvaluateConfigValueError',
'DagsterExecutionStepExecutionError',
'DagsterExecutionStepNotFoundError',
'DagsterExpectationFailedError',
'DagsterInvalidDefinitionError',
'DagsterInvariantViolationError',
'DagsterRuntimeCoercionError',
'DagsterTypeError',
'DagsterUserCodeExecutionError',
'DagsterUserError',
'DagsterStepOutputNotFoundError',
# Utilities
'execute_solid',
@@ -129,6 +137,8 @@
'input_schema',
'input_selector_schema',
'Dict',
'Enum',
'EnumValue',
'Float',
'Int',
'List',
@@ -1,9 +1,4 @@
from dagster import (
DependencyDefinition,
InputDefinition,
PipelineDefinition,
lambda_solid,
)
from dagster import DependencyDefinition, InputDefinition, PipelineDefinition, lambda_solid


@lambda_solid
@@ -4,9 +4,7 @@
from dagster import Field, PipelineDefinition, execute_pipeline, solid, types


@solid(
config_field=Field(types.String, is_optional=True, default_value='en-us')
)
@solid(config_field=Field(types.String, is_optional=True, default_value='en-us'))
def configurable_hello(context):
if len(context.solid_config) >= 3 and context.solid_config[:3] == 'haw':
return 'Aloha honua!'
@@ -17,13 +15,10 @@ def configurable_hello(context):


def define_configurable_hello_pipeline():
return PipelineDefinition(
name='configurable_hello_pipeline', solids=[configurable_hello]
)
return PipelineDefinition(name='configurable_hello_pipeline', solids=[configurable_hello])


def test_intro_tutorial_part_four():
execute_pipeline(
define_configurable_hello_pipeline(),
{'solids': {'configurable_hello': {'config': 'cn'}}},
define_configurable_hello_pipeline(), {'solids': {'configurable_hello': {'config': 'cn'}}}
)
@@ -28,17 +28,13 @@ def count_letters(word):
return dict(counts)


@solid(
inputs=[InputDefinition('word', String)],
config_field=Field(Dict({'factor': Field(Int)})),
)
@solid(inputs=[InputDefinition('word', String)], config_field=Field(Dict({'factor': Field(Int)})))
def typed_multiply_the_word(context, word):
return word * context.solid_config['factor']


@solid(
inputs=[InputDefinition('word', String)],
config_field=Field(Dict({'factor': Field(String)})),
inputs=[InputDefinition('word', String)], config_field=Field(Dict({'factor': Field(String)}))
)
def typed_multiply_the_word_error(context, word):
return word * context.solid_config['factor']
@@ -48,23 +44,15 @@ def define_demo_configuration_schema_pipeline():
return PipelineDefinition(
name='demo_configuration_schema',
solids=[multiply_the_word, count_letters],
dependencies={
'count_letters': {
'word': DependencyDefinition('multiply_the_word')
}
},
dependencies={'count_letters': {'word': DependencyDefinition('multiply_the_word')}},
)


def define_typed_demo_configuration_schema_pipeline():
return PipelineDefinition(
name='typed_demo_configuration_schema',
solids=[typed_multiply_the_word, count_letters],
dependencies={
'count_letters': {
'word': DependencyDefinition('typed_multiply_the_word')
}
},
dependencies={'count_letters': {'word': DependencyDefinition('typed_multiply_the_word')}},
)


@@ -73,9 +61,7 @@ def define_typed_demo_configuration_schema_error_pipeline():
name='typed_demo_configuration_schema_error',
solids=[typed_multiply_the_word_error, count_letters],
dependencies={
'count_letters': {
'word': DependencyDefinition('typed_multiply_the_word_error')
}
'count_letters': {'word': DependencyDefinition('typed_multiply_the_word_error')}
},
)

@@ -18,15 +18,13 @@ def define_execution_context_pipeline_step_one():

def define_execution_context_pipeline_step_two():
return PipelineDefinition(
name='execution_context_pipeline',
solids=[debug_message, error_message],
name='execution_context_pipeline', solids=[debug_message, error_message]
)


def define_execution_context_pipeline_step_three():
return PipelineDefinition(
name='execution_context_pipeline',
solids=[debug_message, error_message],
name='execution_context_pipeline', solids=[debug_message, error_message]
)


@@ -17,9 +17,7 @@
expectations=[
ExpectationDefinition(
name='check_positive',
expectation_fn=lambda _info, value: ExpectationResult(
success=value > 0
),
expectation_fn=lambda _info, value: ExpectationResult(success=value > 0),
)
],
),
@@ -32,6 +30,4 @@ def add_ints(num_one, num_two):


def define_expectations_tutorial_pipeline():
return PipelineDefinition(
name='expectations_tutorial_pipeline', solids=[add_ints]
)
return PipelineDefinition(name='expectations_tutorial_pipeline', solids=[add_ints])
@@ -1,9 +1,4 @@
from dagster import (
DependencyDefinition,
InputDefinition,
PipelineDefinition,
lambda_solid,
)
from dagster import DependencyDefinition, InputDefinition, PipelineDefinition, lambda_solid


@lambda_solid
@@ -20,7 +15,5 @@ def define_hello_dag_pipeline():
return PipelineDefinition(
name='hello_dag_pipeline',
solids=[solid_one, solid_two],
dependencies={
'solid_two': {'arg_one': DependencyDefinition('solid_one')}
},
dependencies={'solid_two': {'arg_one': DependencyDefinition('solid_one')}},
)
@@ -7,9 +7,7 @@ def hello_world():


def define_hello_world_pipeline():
return PipelineDefinition(
name='hello_world_pipeline', solids=[hello_world]
)
return PipelineDefinition(name='hello_world_pipeline', solids=[hello_world])


if __name__ == '__main__':
Oops, something went wrong.

0 comments on commit be5a9cb

Please sign in to comment.
You can’t perform that action at this time.