Skip to content
Permalink
Browse files

[pylint] signature-mutators

Summary: Use signature-mutators to communicate that @solid etc change function args.

Test Plan: lint

Reviewers: #ft, schrockn

Reviewed By: #ft, schrockn

Subscribers: schrockn, natekupp

Differential Revision: https://dagster.phacility.com/D1078
  • Loading branch information...
alangenfeld committed Sep 25, 2019
1 parent 05185d3 commit f240b116e01eb82cb270599a7586984e59ede633
Showing with 74 additions and 163 deletions.
  1. +2 −1 .pylintrc
  2. +1 −1 docs/sections/deploying/dask.md
  3. +1 −1 docs/sections/learn/guides/testing/testing.md
  4. +1 −4 docs/sections/learn/tutorial/hello_world.rst
  5. +0 −1 examples/dagster_examples/airline_demo/pipelines.py
  6. +0 −2 examples/dagster_examples/airline_demo/solids.py
  7. +0 −1 examples/dagster_examples/event_pipeline_demo/pipelines.py
  8. +2 −2 examples/dagster_examples/gcp_data_platform/final_pipeline.py
  9. +1 −5 examples/dagster_examples/gcp_data_platform/resources_pipeline.py
  10. +1 −5 examples/dagster_examples/gcp_data_platform/simple_pipeline.py
  11. +0 −1 examples/dagster_examples/intro_tutorial/actual_dag.py
  12. +1 −1 examples/dagster_examples/intro_tutorial/composition.py
  13. +0 −1 examples/dagster_examples/intro_tutorial/config.py
  14. +0 −2 examples/dagster_examples/intro_tutorial/configuration_schemas.py
  15. +0 −2 examples/dagster_examples/intro_tutorial/configuration_schemas_basic.py
  16. +0 −2 examples/dagster_examples/intro_tutorial/custom_types.py
  17. +0 −2 examples/dagster_examples/intro_tutorial/execution_context.py
  18. +1 −1 examples/dagster_examples/intro_tutorial/expectations.py
  19. +0 −1 examples/dagster_examples/intro_tutorial/hello_dag.py
  20. +0 −1 examples/dagster_examples/intro_tutorial/hello_world.py
  21. +0 −1 examples/dagster_examples/intro_tutorial/inputs.py
  22. +0 −1 examples/dagster_examples/intro_tutorial/multiple_outputs_conditional.py
  23. +0 −1 examples/dagster_examples/intro_tutorial/multiple_outputs_yield.py
  24. +0 −2 examples/dagster_examples/intro_tutorial/pipeline_cli_execution.py
  25. +1 −1 examples/dagster_examples/intro_tutorial/repos.py
  26. +0 −2 examples/dagster_examples/intro_tutorial/resources_full.py
  27. +0 −3 examples/dagster_examples/intro_tutorial/resources_one.py
  28. +1 −1 examples/dagster_examples/jaffle_dbt/jaffle.py
  29. +0 −2 examples/dagster_examples/pyspark_pagerank/pyspark_pagerank_pipeline.py
  30. +0 −2 examples/dagster_examples/pyspark_pagerank/steps/step_five.py
  31. +0 −2 examples/dagster_examples/pyspark_pagerank/steps/step_four.py
  32. +0 −2 examples/dagster_examples/pyspark_pagerank/steps/step_one.py
  33. +0 −2 examples/dagster_examples/pyspark_pagerank/steps/step_three.py
  34. +0 −2 examples/dagster_examples/pyspark_pagerank/steps/step_two.py
  35. +0 −2 examples/dagster_examples/toys/composition.py
  36. +0 −2 examples/dagster_examples/toys/config_mapping.py
  37. +0 −2 examples/dagster_examples/toys/error_monster.py
  38. +1 −1 examples/dagster_examples/toys/fan_in_fan_out.py
  39. +1 −1 examples/dagster_examples/toys/hammer.py
  40. +0 −2 examples/dagster_examples/toys/log_demo.py
  41. +0 −2 examples/dagster_examples/toys/many_events.py
  42. +0 −1 examples/dagster_examples/toys/pandas_hello_world.py
  43. +0 −2 examples/dagster_examples/toys/resources.py
  44. +0 −2 examples/dagster_examples/toys/resources_error.py
  45. +1 −1 examples/dagster_examples/toys/sleepy.py
  46. +1 −3 examples/dagster_examples/toys/stdout_spew.py
  47. +3 −3 examples/dagster_examples_tests/airline_demo_tests/test_types.py
  48. +0 −1 ...ples/dagster_examples_tests/airline_demo_tests/unit_tests/test_ingest_csv_file_handle_to_spark.py
  49. +1 −1 examples/dagster_examples_tests/airline_demo_tests/unit_tests/test_load_data_from_spark.py
  50. +1 −2 examples/dagster_examples_tests/airline_demo_tests/unit_tests/test_unzip_file_handle.py
  51. +1 −1 ...ts/airline_demo_tests/unit_tests/testing_guide_tests/test_cache_file_from_s3_in_guide_step_one.py
  52. +1 −1 ...ts/airline_demo_tests/unit_tests/testing_guide_tests/test_cache_file_from_s3_in_guide_step_two.py
  53. +1 −1 python_modules/dagit/dagit_tests/pipeline.py
  54. +2 −2 python_modules/dagster-airflow/dagster_airflow_tests/test_project/dagster_airflow_demo.py
  55. +2 −2 python_modules/dagster-dask/dagster_dask_tests/test_execute.py
  56. +1 −3 python_modules/dagster-graphql/dagster_graphql_tests/graphql/setup.py
  57. +1 −1 python_modules/dagster-graphql/dagster_graphql_tests/test_cli.py
  58. +5 −5 python_modules/dagster-graphql/dagster_graphql_tests/test_multiprocessing.py
  59. +1 −1 python_modules/dagster/dagster_tests/cli_tests/test_cli_commands.py
  60. +0 −2 python_modules/dagster/dagster_tests/core_tests/definitions_tests/test_composition.py
  61. +0 −1 python_modules/dagster/dagster_tests/core_tests/definitions_tests/test_pipeline.py
  62. +2 −2 ...on_modules/dagster/dagster_tests/core_tests/execution_plan_tests/test_execution_plan_composite.py
  63. +1 −1 python_modules/dagster/dagster_tests/core_tests/storage_tests/test_local_instance.py
  64. +0 −2 python_modules/dagster/dagster_tests/core_tests/system_config_tests/test_system_config.py
  65. +4 −4 python_modules/dagster/dagster_tests/core_tests/test_composites.py
  66. +1 −1 python_modules/dagster/dagster_tests/core_tests/test_mode_definitions.py
  67. +2 −2 python_modules/dagster/dagster_tests/core_tests/test_multi_dependency.py
  68. +0 −2 python_modules/dagster/dagster_tests/core_tests/test_multiple_outputs.py
  69. +0 −2 python_modules/dagster/dagster_tests/core_tests/test_naming_collisions.py
  70. +0 −1 python_modules/dagster/dagster_tests/core_tests/test_pipeline_execution.py
  71. +0 −1 python_modules/dagster/dagster_tests/core_tests/test_solid_with_config.py
  72. +0 −1 python_modules/dagster/dagster_tests/core_tests/types_tests/evaluator_tests/test_config_mappings.py
  73. +0 −1 python_modules/dagster/dagster_tests/core_tests/types_tests/test_config_type_system.py
  74. +1 −1 python_modules/dagster/dagster_tests/core_tests/types_tests/test_types.py
  75. +0 −2 python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_file_handle_to_s3.py
  76. +0 −1 python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_s3_file_manager.py
  77. +1 −1 python_modules/libraries/dagster-datadog/README.md
  78. +1 −3 python_modules/libraries/dagster-gcp/dagster_gcp_tests/bigquery_tests/test_solids.py
  79. +1 −1 python_modules/libraries/dagster-ge/dagster_ge_tests/test_pandas_ge.py
  80. +2 −4 python_modules/libraries/dagster-pandas/dagster_pandas/examples/pandas_hello_world/pipeline.py
  81. +10 −10 python_modules/libraries/dagster-pandas/dagster_pandas_tests/test_config_driven_df.py
  82. +1 −1 python_modules/libraries/dagster-papertrail/README.md
  83. +1 −1 python_modules/libraries/dagster-papertrail/dagster_papertrail_tests/test_loggers.py
  84. +9 −9 python_modules/libraries/dagster-postgres/dagster_postgres_tests/test_event_log.py
@@ -19,7 +19,8 @@ disable=C,R,duplicate-code,W0511,W1201,W1202,no-init

# See: https://github.com/getsentry/responses/issues/74
[TYPECHECK]
ignored-classes= responses
ignored-classes=responses
signature-mutators=solid,composite_solid,lambda_solid

[MASTER]
ignore=snapshots
@@ -45,7 +45,7 @@ def hello_world(_):
@pipeline(mode_defs=ModeDefinition(executor_defs=default_executors + [dask_executor]))
def dask_pipeline():
return hello_world() # pylint: disable=no-value-for-parameter
return hello_world()
execute_pipeline(
@@ -89,7 +89,7 @@ def test_cache_file_from_s3_step_one_one():
@pipeline
def pipe():
# pylint: disable=no-value-for-parameter
return cache_file_from_s3(emit_value())
execute_pipeline(pipe)
@@ -22,10 +22,7 @@ This example introduces the core **solid** and **pipeline** concepts:

2. A **pipeline** is a set of solids arranged into a DAG of computation that produces data assets.
In this example, the function decorated call to :py:func:`@pipeline <dagster.pipeline>` defines
a pipeline with a single solid. The solid is not actually invoked during pipeline construction.
The invocation places that solid within the pipeline. Context arguments are only passed when the
solids are actually executed, and so we disable the pylint warning with the
``# pylint: disable=no-value-for-parameter`` pylint directive within the comment.
a pipeline with a single solid.

Pipeline Execution
^^^^^^^^^^^^^^^^^^
@@ -1,4 +1,3 @@
# pylint: disable=no-value-for-parameter
"""Pipeline definitions for the airline_demo."""
from dagster_aws.s3.file_cache import s3_file_cache
from dagster_aws.s3.file_manager import S3FileHandle
@@ -244,7 +244,6 @@ def subsample_spark_dataset(context, data_frame: DataFrame) -> DataFrame:
more detail.'''
)
def s3_to_df(s3_coordinate: S3Coordinate, archive_member: String) -> DataFrame:
# pylint: disable=no-value-for-parameter
return ingest_csv_file_handle_to_spark(
unzip_file_handle(cache_file_from_s3(s3_coordinate), archive_member)
)
@@ -263,7 +262,6 @@ def s3_to_df(s3_coordinate: S3Coordinate, archive_member: String) -> DataFrame:
''',
)
def s3_to_dw_table(s3_coordinate: S3Coordinate, archive_member: String) -> String:
# pylint: disable=no-value-for-parameter
return load_data_to_database_from_spark(
canonicalize_column_names(subsample_spark_dataset(s3_to_df(s3_coordinate, archive_member)))
)
@@ -168,5 +168,4 @@ def snowflake_load(context):
logger=context.log,
)

# pylint: disable=no-value-for-parameter
snowflake_load(event_ingest(spark_inputs=gunzipper(gzip_file=download_from_s3_to_file())))
@@ -90,7 +90,7 @@ def events_dataproc_fn(context, cfg):
},
)
def events_dataproc() -> List[String]:
return output_paths(dataproc_solid()) # pylint: disable=no-value-for-parameter
return output_paths(dataproc_solid())


def bq_load_events_fn(context, cfg):
@@ -116,7 +116,7 @@ def bq_load_events_fn(context, cfg):

@composite_solid(config_fn=bq_load_events_fn, config={'table': Field(String)})
def bq_load_events(source_uris: List[String]):
return import_gcs_paths_to_bq(source_uris) # pylint: disable=no-value-for-parameter
return import_gcs_paths_to_bq(source_uris)


def explore_visits_by_hour_fn(_, cfg):
@@ -124,11 +124,7 @@ def explore_visits_by_hour(context):
],
)
def gcp_data_platform():
dataproc_job = delete_dataproc_cluster(
data_proc_spark_operator(
create_dataproc_cluster() # pylint: disable=no-value-for-parameter
)
)
dataproc_job = delete_dataproc_cluster(data_proc_spark_operator(create_dataproc_cluster()))

events_in_bq = gcs_to_bigquery(dataproc_job)
explore_visits_by_hour(events_in_bq)
@@ -121,11 +121,7 @@ def explore_visits_by_hour(_):

@pipeline
def gcp_data_platform():
dataproc_job = delete_dataproc_cluster(
data_proc_spark_operator(
create_dataproc_cluster() # pylint: disable=no-value-for-parameter
)
)
dataproc_job = delete_dataproc_cluster(data_proc_spark_operator(create_dataproc_cluster()))

events_in_bq = gcs_to_bigquery(dataproc_job)
explore_visits_by_hour(events_in_bq)
@@ -1,4 +1,3 @@
# pylint: disable=no-value-for-parameter
from dagster import pipeline, solid


@@ -1,4 +1,4 @@
# pylint: disable=no-value-for-parameter,unused-variable,no-member
# pylint:disable=unused-variable,no-member

from dagster import Output, OutputDefinition, composite_solid, pipeline, solid

@@ -1,4 +1,3 @@
# pylint: disable=no-value-for-parameter
# encoding: utf-8

from dagster import Field, execute_pipeline, pipeline, solid
@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

import collections

from dagster import Field, Int, as_dagster_type, pipeline, solid
@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

import collections

from dagster import Any, Field, as_dagster_type, pipeline, solid
@@ -12,8 +12,6 @@
)
from dagster.utils import script_relative_path

# pylint: disable=no-value-for-parameter


@dagster_type
class Empty:
@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from dagster import pipeline, solid


@@ -10,4 +10,4 @@ def add_ints(_, num_one: int, num_two: int) -> int:

@pipeline
def expectations_tutorial_pipeline():
add_ints() # pylint: disable=no-value-for-parameter
add_ints()
@@ -1,4 +1,3 @@
# pylint: disable=no-value-for-parameter
from dagster import pipeline, solid


@@ -1,4 +1,3 @@
# pylint: disable=no-value-for-parameter
from dagster import execute_pipeline, pipeline, solid


@@ -1,4 +1,3 @@
# pylint: disable=no-value-for-parameter
from dagster import String, execute_pipeline, pipeline, solid


@@ -1,4 +1,3 @@
# pylint: disable=no-value-for-parameter
from dagster import Field, Int, Output, OutputDefinition, String, pipeline, solid


@@ -1,4 +1,3 @@
# pylint: disable=no-value-for-parameter
from dagster import Int, Output, OutputDefinition, pipeline, solid


@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from collections import defaultdict

from dagster import Field, Int, RepositoryDefinition, pipeline, solid
@@ -8,7 +8,7 @@ def hello_world(_):

@pipeline
def repo_demo_pipeline():
hello_world() # pylint: disable=no-value-for-parameter
hello_world()


def define_repo():
@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from dagster_slack import slack_resource

from dagster import (
@@ -1,6 +1,3 @@
# pylint: disable=no-value-for-parameter


from dagster_slack import slack_resource

from dagster import ModeDefinition, execute_pipeline, pipeline, solid
@@ -13,4 +13,4 @@

@pipeline
def jaffle_pipeline():
jaffle_test_solid(jaffle_solid()) # pylint: disable=no-value-for-parameter
jaffle_test_solid(jaffle_solid())
@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from operator import add

from dagster_pyspark import SparkRDD, spark_session_resource
@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

import re
from operator import add

@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

import re
from operator import add

@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from dagster import pipeline, solid


@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

import re
from operator import add

@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

import re
from operator import add

@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from dagster import InputDefinition, Int, composite_solid, pipeline, solid


@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from dagster import Field, Int, String, composite_solid, pipeline, solid


@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from dagster import (
Bool,
Dict,
@@ -27,7 +27,7 @@ def construct_fan_in_level(source, level, fanout):
def construct_level_pipeline(name, levels, fanout):
@pipeline(name=name)
def _pipe():
# pylint: disable=no-value-for-parameter

return_one_out = return_one()
prev_level_out = return_one_out
for level in range(0, levels):
@@ -102,7 +102,7 @@ def reducer(_, in_1, in_2, in_3, in_4):
]
)
def hammer_pipeline():
# pylint: disable=no-value-for-parameter

out_1, out_2, out_3, out_4 = chase_giver()
return reducer(
in_1=hammer.alias('hammer_1')(chase_duration=out_1),
@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from dagster import pipeline, solid


@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from dagster import (
EventMetadataEntry,
ExpectationResult,
@@ -1,4 +1,3 @@
# pylint: disable=no-value-for-parameter
import pandas as pd
from dagster_pandas import DataFrame

@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from dagster import (
ExecutionTargetHandle,
Field,
@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

from dagster import Field, Int, ModeDefinition, execute_pipeline, pipeline, resource, solid


@@ -1,4 +1,4 @@
# pylint: disable=no-value-for-parameter, no-member
# pylint:disable=no-member

from time import sleep

@@ -1,5 +1,3 @@
# pylint: disable=no-value-for-parameter

import os
import sys
import time
@@ -38,6 +36,6 @@ def spew(_, name):

@pipeline(mode_defs=[ModeDefinition(executor_defs=default_executors)])
def stdout_spew_pipeline():
out_1, out_2 = spawn() # pylint: disable=no-value-for-parameter
out_1, out_2 = spawn()
spew.alias('spew_1')(name=out_1)
spew.alias('spew_2')(name=out_2)
@@ -32,7 +32,7 @@ def nonce(_):

@pipeline(mode_defs=[spark_mode])
def spark_df_test_pipeline():
# pylint: disable=no-value-for-parameter

ingest_csv_file_handle_to_spark(nonce())

run_id = str(uuid.uuid4())
@@ -71,7 +71,7 @@ def nonce(context):

@pipeline(mode_defs=[spark_mode])
def spark_df_test_pipeline():
# pylint: disable=no-value-for-parameter

ingest_csv_file_handle_to_spark(nonce())

run_id = str(uuid.uuid4())
@@ -123,7 +123,7 @@ def passthrough_df(_context, df):

@pipeline
def passthrough():
passthrough_df(emit()) # pylint: disable=no-value-for-parameter
passthrough_df(emit())

with seven.TemporaryDirectory() as tempdir:
file_name = os.path.join(tempdir, 'output.csv')
@@ -6,7 +6,6 @@
from dagster.utils import file_relative_path

# for dep graphs
# pylint: disable=no-value-for-parameter


def test_ingest_csv_file_handle_to_spark():
@@ -29,7 +29,7 @@ def emit_mock(_):
]
)
def load_df_test():
load_data_to_database_from_spark(emit_mock()) # pylint: disable=no-value-for-parameter
load_data_to_database_from_spark(emit_mock())

solid_result = execute_pipeline(
load_df_test,

0 comments on commit f240b11

Please sign in to comment.
You can’t perform that action at this time.