Skip to content
Permalink
Browse files

Incremental improvements and fixes to tutorial (#592)

  • Loading branch information...
mgasner committed Jan 10, 2019
1 parent 247121c commit 3df75e57d4bbf0dc02f5ec3344a90b454bd7c721
Showing with 762 additions and 626 deletions.
  1. +3 −0 .gitignore
  2. +2 −0 pyproject.toml
  3. +3 −3 python_modules/dagit/dagit_tests/test_smoke.py
  4. +28 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/config.py
  5. +3 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/config_env.yml
  6. +88 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/configuration_schemas.py
  7. +9 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/configuration_schemas.yml
  8. +9 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/configuration_schemas_error_1.yml
  9. +9 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/configuration_schemas_error_2.yml
  10. +4 −3 python_modules/dagster/dagster/tutorials/intro_tutorial/{part_five.py → execution_context.py}
  11. 0 python_modules/dagster/dagster/tutorials/intro_tutorial/{part_five.yaml → execution_context.yml}
  12. +3 −3 python_modules/dagster/dagster/tutorials/intro_tutorial/inputs.py
  13. +1 −1 python_modules/dagster/dagster/tutorials/intro_tutorial/inputs_env.yml
  14. +0 −19 python_modules/dagster/dagster/tutorials/intro_tutorial/part_four.py
  15. +0 −3 python_modules/dagster/dagster/tutorials/intro_tutorial/part_four_env.yml
  16. +0 −42 python_modules/dagster/dagster/tutorials/intro_tutorial/part_seven.py
  17. +0 −3 python_modules/dagster/dagster/tutorials/intro_tutorial/part_seven_repository.yml
  18. +0 −17 python_modules/dagster/dagster/tutorials/intro_tutorial/part_six.py
  19. +0 −3 python_modules/dagster/dagster/tutorials/intro_tutorial/part_six_repository_1.yml
  20. +0 −3 python_modules/dagster/dagster/tutorials/intro_tutorial/part_six_repository_2.yml
  21. +10 −2 python_modules/dagster/dagster/tutorials/intro_tutorial/{part_eight.py → pipeline_execution.py}
  22. 0 ...odules/dagster/dagster/tutorials/intro_tutorial/{part_seven_env.yml → pipeline_execution_env.yml}
  23. +3 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/pipeline_execution_repository.yml
  24. +2 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/pyproject.toml
  25. +17 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/repos.py
  26. +3 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/repos_1.yml
  27. +3 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/repos_2.yml
  28. +38 −0 python_modules/dagster/dagster/tutorials/intro_tutorial/test/test_config.py
  29. +6 −10 ...les/dagster/dagster/tutorials/intro_tutorial/test/{test_part_five.py → test_execution_context.py}
  30. +1 −1 python_modules/dagster/dagster/tutorials/intro_tutorial/test/test_inputs.py
  31. +0 −37 python_modules/dagster/dagster/tutorials/intro_tutorial/test/test_part_four.py
  32. +4 −4 python_modules/dagster/dagster/tutorials/intro_tutorial/tutorial_repository.py
  33. +0 −3 python_modules/dagster/dagster/tutorials/repository.yml
  34. +0 −6 python_modules/dagster/dagster/tutorials/test_intro_tutorial_part_eleven.py
  35. +40 −40 ..._modules/dagster/dagster/tutorials/test_intro_tutorial_part_nine/test_intro_tutorial_part_nine.py
  36. +9 −9 python_modules/dagster/dagster/tutorials/test_intro_tutorial_part_ten.py
  37. +2 −2 python_modules/dagster/dagster_tests/cli_tests/repository.yml
  38. +2 −2 python_modules/dagster/dagster_tests/cli_tests/repository_module.yml
  39. +11 −11 python_modules/dagster/dagster_tests/cli_tests/test_cli_commands.py
  40. +16 −16 python_modules/dagster/dagster_tests/cli_tests/test_dynamic_loader.py
  41. +5 −5 python_modules/dagster/docs/index.rst
  42. +6 −5 python_modules/dagster/docs/intro_tutorial/actual_dag.rst
  43. BIN python_modules/dagster/docs/intro_tutorial/{actual_dag_fig_one.png → actual_dag_figure_one.png}
  44. +72 −0 python_modules/dagster/docs/intro_tutorial/config.rst
  45. BIN python_modules/dagster/docs/intro_tutorial/config_figure_one.png
  46. +101 −0 python_modules/dagster/docs/intro_tutorial/configuration_schemas.rst
  47. +101 −0 python_modules/dagster/docs/intro_tutorial/execution_context.rst
  48. +7 −4 python_modules/dagster/docs/intro_tutorial/hello_dag.rst
  49. BIN python_modules/dagster/docs/intro_tutorial/{hello_dag_fig_one.png → hello_dag_figure_one.png}
  50. +16 −9 python_modules/dagster/docs/intro_tutorial/hello_world.rst
  51. BIN python_modules/dagster/docs/intro_tutorial/{hello_world_fig_one.png → hello_world_figure_one.png}
  52. BIN python_modules/dagster/docs/intro_tutorial/{hello_world_fig_two.png → hello_world_figure_two.png}
  53. +45 −27 python_modules/dagster/docs/intro_tutorial/inputs.rst
  54. +0 −121 python_modules/dagster/docs/intro_tutorial/part_eight.rst
  55. +0 −94 python_modules/dagster/docs/intro_tutorial/part_five.rst
  56. +0 −66 python_modules/dagster/docs/intro_tutorial/part_four.rst
  57. BIN python_modules/dagster/docs/intro_tutorial/part_four_fig_one.png
  58. +0 −31 python_modules/dagster/docs/intro_tutorial/part_seven.rst
  59. +58 −0 python_modules/dagster/docs/intro_tutorial/pipeline_execution.rst
  60. +22 −21 python_modules/dagster/docs/intro_tutorial/{part_six.rst → repos.rst}
  61. BIN python_modules/dagster/docs/intro_tutorial/{part_six_fig_one.png → repos_figure_one.png}
@@ -112,3 +112,6 @@ python_modules/dagster/docs/_build
.vscode/

dagit_run_logs

python_modules/dagit/node_modules/
python_modules/dagit/yarn.lock
@@ -0,0 +1,2 @@
[tool.black]
line-length = 100
@@ -2,11 +2,11 @@
import json
from dagit import app
from dagit.pipeline_run_storage import PipelineRunStorage
from dagster.tutorials.intro_tutorial.part_six import define_part_six_repo
from dagster.tutorials.intro_tutorial.repos import define_repo


def test_smoke_app():
repository_container = app.RepositoryContainer(repository=define_part_six_repo())
repository_container = app.RepositoryContainer(repository=define_repo())
pipeline_run_storage = PipelineRunStorage()
flask_app = app.create_app(repository_container, pipeline_run_storage)
client = flask_app.test_client()
@@ -18,5 +18,5 @@ def test_smoke_app():
assert len(data['data']['pipelines']['nodes']) == 1

assert set([node_data['name'] for node_data in data['data']['pipelines']['nodes']]) == set(
['part_six']
['repo_demo_pipeline']
)
@@ -0,0 +1,28 @@
# encoding: utf-8

from dagster import Field, PipelineDefinition, execute_pipeline, solid, types


@solid(
config_field=Field(types.String, is_optional=True, default_value='en-us')
)
def configurable_hello(info):
if len(info.config) >= 3 and info.config[:3] == 'haw':
return 'Aloha honua!'
elif len(info.config) >= 2 and info.config[:2] == 'cn':
return '你好, 世界!'
else:
return 'Hello, world!'


def define_configurable_hello_pipeline():
return PipelineDefinition(
name='configurable_hello_pipeline', solids=[configurable_hello]
)


def test_intro_tutorial_part_four():
execute_pipeline(
define_configurable_hello_pipeline(),
{'solids': {'configurable_hello': {'config': 'cn'}}},
)
@@ -0,0 +1,3 @@
solids:
configurable_hello:
config: "haw"
@@ -0,0 +1,88 @@
from collections import defaultdict

from dagster import (
DependencyDefinition,
InputDefinition,
lambda_solid,
OutputDefinition,
PipelineDefinition,
RepositoryDefinition,
solid,
types,
)


@solid(
config_field=types.Field(types.Dict({'word': types.Field(types.String)}))
)
def double_the_word(info):
return info.config['word'] * 2


@lambda_solid(inputs=[InputDefinition('word')])
def count_letters(word):
counts = defaultdict(int)
for letter in word:
counts[letter] += 1
return dict(counts)


@solid(
config_field=types.Field(types.Dict({'word': types.Field(types.String)})),
outputs=[OutputDefinition(types.String)],
)
def typed_double_the_word(info):
return info.config['word'] * 2


@solid(
config_field=types.Field(types.Dict({'word': types.Field(types.String)})),
outputs=[OutputDefinition(types.Int)],
)
def typed_double_the_word_error(info):
return info.config['word'] * 2


def define_demo_configuration_schema_pipeline():
return PipelineDefinition(
name='demo_configuration_schema',
solids=[double_the_word, count_letters],
dependencies={
'count_letters': {'word': DependencyDefinition('double_the_word')}
},
)


def define_typed_demo_configuration_schema_pipeline():
return PipelineDefinition(
name='typed_demo_configuration_schema',
solids=[typed_double_the_word, count_letters],
dependencies={
'count_letters': {
'word': DependencyDefinition('typed_double_the_word')
}
},
)


def define_typed_demo_configuration_schema_error_pipeline():
return PipelineDefinition(
name='typed_demo_configuration_schema_error',
solids=[typed_double_the_word_error, count_letters],
dependencies={
'count_letters': {
'word': DependencyDefinition('typed_double_the_word_error')
}
},
)


def define_demo_configuration_schema_repo():
return RepositoryDefinition(
name='demo_configuration_schema_repo',
pipeline_dict={
'demo_configuration_schema': define_demo_configuration_schema_pipeline,
'typed_demo_configuration_schema': define_typed_demo_configuration_schema_pipeline,
'typed_demo_configuration_schema_error': define_typed_demo_configuration_schema_error_pipeline,
},
)
@@ -0,0 +1,9 @@
context:
default:
config:
log_level: DEBUG

solids:
double_the_word:
config:
word: quux
@@ -0,0 +1,9 @@
context:
default:
config:
log_level: DEBUG

solids:
double_the_word:
config:
word: 1
@@ -0,0 +1,9 @@
context:
default:
config:
log_level: DEBUG

solids:
double_the_word_with_typed_config:
config:
wrong_word: quux
@@ -10,7 +10,6 @@ def debug_message(info):
@solid
def error_message(info):
info.context.error('An error occurred.')
raise Exception()


def define_execution_context_pipeline_step_one():
@@ -19,13 +18,15 @@ def define_execution_context_pipeline_step_one():

def define_execution_context_pipeline_step_two():
return PipelineDefinition(
name='part_five_pipeline', solids=[debug_message, error_message]
name='execution_context_pipeline',
solids=[debug_message, error_message],
)


def define_execution_context_pipeline_step_three():
return PipelineDefinition(
name='part_five_pipeline', solids=[debug_message, error_message]
name='execution_context_pipeline',
solids=[debug_message, error_message],
)


@@ -20,13 +20,13 @@ def define_hello_inputs_pipeline():
def execute_with_another_world():
return execute_pipeline(
define_hello_inputs_pipeline(),
# This entire dictionary is known as the "environment".
# This entire dictionary is known as the 'environment'.
# It has many sections.
{
# This is the "solids" section
# This is the 'solids' section
'solids': {
# Configuration for the add_hello_to_word solid
'add_hello_to_word': {'inputs': {'word': 'Another World'}}
'add_hello_to_word': {'inputs': {'word': 'Mars'}}
}
},
)
@@ -1,4 +1,4 @@
solids:
add_hello_to_word:
inputs:
word: "Another World"
word: "Mars"

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.
@@ -6,6 +6,7 @@
Field,
InputDefinition,
PipelineDefinition,
RepositoryDefinition,
String,
lambda_solid,
solid,
@@ -25,11 +26,18 @@ def count_letters(word):
return dict(counts)


def define_part_eight_step_one_pipeline():
def define_demo_execution_pipeline():
return PipelineDefinition(
name='part_eight_step_one_pipeline',
name='demo_execution',
solids=[double_the_word, count_letters],
dependencies={
'count_letters': {'word': DependencyDefinition('double_the_word')}
},
)


def define_demo_execution_repo():
return RepositoryDefinition(
name='demo_execution_repo',
pipeline_dict={'part_seven': define_demo_execution_pipeline},
)
@@ -0,0 +1,3 @@
repository:
file: pipeline_execution.py
fn: define_demo_execution_repo
@@ -0,0 +1,2 @@
[tool.black]
line-length = 80
@@ -0,0 +1,17 @@
from dagster import lambda_solid, PipelineDefinition, RepositoryDefinition


@lambda_solid
def hello_world():
pass


def define_repo_demo_pipeline():
return PipelineDefinition(name='repo_demo_pipeline', solids=[hello_world])


def define_repo():
return RepositoryDefinition(
name='demo_repository',
pipeline_dict={'repo_demo_pipeline': define_repo_demo_pipeline},
)
@@ -0,0 +1,3 @@
repository:
file: repos.py
fn: define_repo
@@ -0,0 +1,3 @@
repository:
module: dagster.tutorials.intro_tutorial.repos
fn: define_repo
Oops, something went wrong.

0 comments on commit 3df75e5

Please sign in to comment.
You can’t perform that action at this time.