Skip to content
Permalink
Browse files

fix isort

Test Plan: should fail bk before i update with isort run

Reviewers: #ft, schrockn

Reviewed By: #ft, schrockn

Differential Revision: https://dagster.phacility.com/D886
  • Loading branch information...
alangenfeld committed Aug 26, 2019
1 parent 09932e2 commit 49ec98229538eea1c11da1352f22170602ce7802
Showing with 490 additions and 548 deletions.
  1. +3 −2 .buildkite/deploy.py
  2. +0 −1 .buildkite/images/integration/trigger_maven.py
  3. +3 −2 .buildkite/nightly.py
  4. +2 −4 .buildkite/pipeline.py
  5. +2 −1 .buildkite/step_builder.py
  6. +2 −2 .isort.cfg
  7. +1 −1 Makefile
  8. +1 −1 bin/create_library.py
  9. +0 −1 bin/publish.py
  10. +1 −2 docs/sections/learn/guides/data_science/iris_pipeline_2.py
  11. +1 −2 docs/sections/learn/guides/data_science/iris_pipeline_3.py
  12. +2 −2 examples/dagster_examples/airline_demo/cache_file_from_s3.py
  13. +8 −11 examples/dagster_examples/airline_demo/pipelines.py
  14. +2 −1 examples/dagster_examples/airline_demo/resources.py
  15. +5 −6 examples/dagster_examples/airline_demo/solids.py
  16. +1 −1 examples/dagster_examples/airline_demo/unzip_file_handle.py
  17. +5 −5 examples/dagster_examples/dep_dsl/pipeline.py
  18. +5 −6 examples/dagster_examples/event_pipeline_demo/pipelines.py
  19. +12 −12 examples/dagster_examples/gcp_data_platform/final_pipeline.py
  20. +4 −5 examples/dagster_examples/gcp_data_platform/resources_pipeline.py
  21. +2 −3 examples/dagster_examples/gcp_data_platform/simple_pipeline.py
  22. +1 −1 examples/dagster_examples/intro_tutorial/config.py
  23. +1 −1 examples/dagster_examples/intro_tutorial/configuration_schemas.py
  24. +1 −1 examples/dagster_examples/intro_tutorial/configuration_schemas_basic.py
  25. +1 −1 examples/dagster_examples/intro_tutorial/expectations.py
  26. +1 −1 examples/dagster_examples/intro_tutorial/hello_dag.py
  27. +1 −1 examples/dagster_examples/intro_tutorial/hello_world.py
  28. +1 −1 examples/dagster_examples/intro_tutorial/inputs.py
  29. +1 −1 examples/dagster_examples/intro_tutorial/multiple_outputs_conditional.py
  30. +1 −1 examples/dagster_examples/intro_tutorial/multiple_outputs_yield.py
  31. +1 −1 examples/dagster_examples/intro_tutorial/pipeline_cli_execution.py
  32. +1 −1 examples/dagster_examples/intro_tutorial/repos.py
  33. +2 −2 examples/dagster_examples/intro_tutorial/resources_full.py
  34. +2 −2 examples/dagster_examples/intro_tutorial/resources_one.py
  35. +3 −3 examples/dagster_examples/intro_tutorial/tutorial_repository.py
  36. +2 −2 examples/dagster_examples/jaffle_dbt/jaffle.py
  37. +2 −2 examples/dagster_examples/pandas_hello_world/repo.py
  38. +5 −5 examples/dagster_examples/pyspark_pagerank/pyspark_pagerank_pipeline.py
  39. +3 −3 examples/dagster_examples/pyspark_pagerank/repository.py
  40. +4 −4 examples/dagster_examples/pyspark_pagerank/steps/step_five.py
  41. +3 −2 examples/dagster_examples/pyspark_pagerank/steps/step_four.py
  42. +2 −1 examples/dagster_examples/pyspark_pagerank/steps/step_three.py
  43. +1 −1 examples/dagster_examples/pyspark_pagerank/steps/step_two.py
  44. +1 −1 examples/dagster_examples/toys/config_mapping.py
  45. +1 −1 examples/dagster_examples/toys/error_monster.py
  46. +1 −1 examples/dagster_examples/toys/fan_in_fan_out.py
  47. +5 −6 examples/dagster_examples/toys/hammer.py
  48. +1 −1 examples/dagster_examples/toys/log_spew.py
  49. +6 −6 examples/dagster_examples/toys/many_events.py
  50. +2 −1 examples/dagster_examples/toys/pandas_hello_world.py
  51. +3 −3 examples/dagster_examples/toys/repo.py
  52. +2 −2 examples/dagster_examples/toys/resources.py
  53. +1 −1 examples/dagster_examples/toys/resources_error.py
  54. +4 −4 examples/dagster_examples/toys/sleepy.py
  55. +1 −1 examples/dagster_examples/util.py
  56. +4 −5 examples/dagster_examples_tests/airline_demo_tests/test_airflow.py
  57. +1 −3 examples/dagster_examples_tests/airline_demo_tests/test_pipelines.py
  58. +4 −7 examples/dagster_examples_tests/airline_demo_tests/test_solids.py
  59. +10 −13 examples/dagster_examples_tests/airline_demo_tests/test_types.py
  60. +5 −7 examples/dagster_examples_tests/airline_demo_tests/unit_tests/test_cache_file_from_s3.py
  61. +4 −4 ...ples/dagster_examples_tests/airline_demo_tests/unit_tests/test_ingest_csv_file_handle_to_spark.py
  62. +2 −2 examples/dagster_examples_tests/airline_demo_tests/unit_tests/test_load_data_from_spark.py
  63. +5 −4 examples/dagster_examples_tests/airline_demo_tests/unit_tests/test_unzip_file_handle.py
  64. +0 −1 ...tests/unit_tests/testing_guide_tests/snapshots/snap_test_cache_file_from_s3_in_guide_step_four.py
  65. +4 −3 ...s/airline_demo_tests/unit_tests/testing_guide_tests/test_cache_file_from_s3_in_guide_step_four.py
  66. +4 −4 ...ts/airline_demo_tests/unit_tests/testing_guide_tests/test_cache_file_from_s3_in_guide_step_one.py
  67. +5 −5 .../airline_demo_tests/unit_tests/testing_guide_tests/test_cache_file_from_s3_in_guide_step_three.py
  68. +5 −8 ...ts/airline_demo_tests/unit_tests/testing_guide_tests/test_cache_file_from_s3_in_guide_step_two.py
  69. +2 −1 examples/dagster_examples_tests/dep_dsl_tests/test_basic.py
  70. +4 −5 examples/dagster_examples_tests/event_pipeline_demo_tests/test_airflowized.py
  71. +1 −1 examples/dagster_examples_tests/event_pipeline_demo_tests/test_pipelines.py
  72. +0 −1 examples/dagster_examples_tests/graphql_tests/snapshots/snap_test_examples_presets_graphql.py
  73. +1 −0 examples/dagster_examples_tests/graphql_tests/test_examples_presets_graphql.py
  74. +3 −2 examples/dagster_examples_tests/graphql_tests/util.py
  75. +1 −0 examples/dagster_examples_tests/pyspark_pagerank_tests/test_original.py
  76. +2 −2 examples/dagster_examples_tests/pyspark_pagerank_tests/test_pyspark_pagerank.py
  77. +0 −1 examples/dagster_examples_tests/test_toys/snapshots/snap_test_toys.py
  78. +2 −1 examples/dagster_examples_tests/test_toys/test_fan_in_fan_out.py
  79. +2 −1 examples/dagster_examples_tests/test_toys/test_pandas_hello_world.py
  80. +8 −9 examples/dagster_examples_tests/test_toys/test_toys.py
  81. +2 −1 examples/dagster_examples_tests/tutorial_tests/test_actual_dag.py
  82. +1 −1 examples/dagster_examples_tests/tutorial_tests/test_cli_invocations.py
  83. +4 −3 examples/dagster_examples_tests/tutorial_tests/test_composites.py
  84. +2 −1 examples/dagster_examples_tests/tutorial_tests/test_config.py
  85. +6 −6 examples/dagster_examples_tests/tutorial_tests/test_configuration_schemas.py
  86. +2 −1 examples/dagster_examples_tests/tutorial_tests/test_execution_context.py
  87. +2 −1 examples/dagster_examples_tests/tutorial_tests/test_expectations.py
  88. +2 −1 examples/dagster_examples_tests/tutorial_tests/test_hello_dag.py
  89. +2 −1 examples/dagster_examples_tests/tutorial_tests/test_hello_world.py
  90. +4 −4 examples/dagster_examples_tests/tutorial_tests/test_inputs.py
  91. +3 −4 examples/dagster_examples_tests/tutorial_tests/test_multiple_outputs.py
  92. +2 −1 examples/dagster_examples_tests/tutorial_tests/test_repos.py
  93. +3 −2 examples/dagster_examples_tests/tutorial_tests/test_resources.py
  94. +2 −2 examples/dagster_examples_tests/tutorial_tests/test_reusing_solids.py
  95. +0 −3 python_modules/Makefile
  96. +7 −8 python_modules/dagit/dagit/app.py
  97. +1 −1 python_modules/dagit/dagit/cli.py
  98. +1 −0 python_modules/dagit/dagit/dagit.py
  99. +2 −3 python_modules/dagit/dagit_tests/test_app.py
  100. +0 −1 python_modules/dagit/dagit_tests/test_cli.py
  101. +2 −2 python_modules/dagit/dagit_tests/test_smoke.py
  102. +1 −2 python_modules/dagster-airflow/dagster_airflow/factory.py
  103. +2 −5 python_modules/dagster-airflow/dagster_airflow/operators/docker_operator.py
  104. +2 −2 python_modules/dagster-airflow/dagster_airflow/operators/kubernetes_operator.py
  105. +2 −2 python_modules/dagster-airflow/dagster_airflow/operators/python_operator.py
  106. +1 −1 python_modules/dagster-airflow/dagster_airflow/operators/util.py
  107. +2 −2 python_modules/dagster-airflow/dagster_airflow/vendor/docker_operator.py
  108. +2 −2 python_modules/dagster-airflow/dagster_airflow/vendor/kubernetes_pod_operator.py
  109. +0 −1 python_modules/dagster-airflow/dagster_airflow_tests/test_build_dags.py
  110. +3 −3 python_modules/dagster-airflow/dagster_airflow_tests/test_compile.py
  111. +0 −1 python_modules/dagster-airflow/dagster_airflow_tests/test_dagster_docker_operator.py
  112. +3 −4 python_modules/dagster-airflow/dagster_airflow_tests/test_factory.py
  113. +1 −2 python_modules/dagster-airflow/dagster_airflow_tests/test_format_config.py
  114. +3 −3 python_modules/dagster-airflow/dagster_airflow_tests/test_project/dagster_airflow_demo.py
  115. +1 −3 python_modules/dagster-dask/dagster_dask/engine.py
  116. +3 −3 python_modules/dagster-dask/dagster_dask_tests/test_execute.py
  117. +2 −2 python_modules/dagster-graphql/dagster_graphql/client/mutations.py
  118. +1 −1 python_modules/dagster-graphql/dagster_graphql/implementation/context.py
  119. +2 −4 python_modules/dagster-graphql/dagster_graphql/implementation/execution.py
  120. +3 −3 python_modules/dagster-graphql/dagster_graphql/implementation/fetch_types.py
  121. +2 −2 python_modules/dagster-graphql/dagster_graphql/schema/config_types.py
  122. +2 −2 python_modules/dagster-graphql/dagster_graphql/schema/errors.py
  123. +3 −3 python_modules/dagster-graphql/dagster_graphql/schema/execution.py
  124. +2 −2 python_modules/dagster-graphql/dagster_graphql/schema/pipelines.py
  125. +5 −5 python_modules/dagster-graphql/dagster_graphql/schema/roots.py
  126. +1 −1 python_modules/dagster-graphql/dagster_graphql/schema/run_schedule.py
  127. +3 −4 python_modules/dagster-graphql/dagster_graphql/schema/runs.py
  128. +2 −2 python_modules/dagster-graphql/dagster_graphql/schema/runtime_types.py
  129. +2 −2 python_modules/dagster-graphql/dagster_graphql/schema/solids.py
  130. +1 −2 python_modules/dagster-graphql/dagster_graphql/test/utils.py
  131. +2 −2 python_modules/dagster-graphql/dagster_graphql_tests/client_tests/test_mutations.py
  132. +4 −4 python_modules/dagster-graphql/dagster_graphql_tests/client_tests/test_util.py
  133. +4 −3 python_modules/dagster-graphql/dagster_graphql_tests/graphql/setup.py
  134. +0 −1 python_modules/dagster-graphql/dagster_graphql_tests/graphql/snapshots/snap_test_expectations.py
  135. +0 −1 python_modules/dagster-graphql/dagster_graphql_tests/graphql/snapshots/snap_test_resources.py
  136. +2 −2 python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_config_types.py
  137. +1 −2 python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_execute_pipeline.py
  138. +2 −2 python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_execution_plan.py
  139. +5 −4 python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_misc.py
  140. +0 −1 python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_mode_definitions.py
  141. +2 −1 python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_scheduler.py
  142. +2 −2 python_modules/dagster-graphql/dagster_graphql_tests/test_cli.py
  143. +4 −4 python_modules/dagster-graphql/dagster_graphql_tests/test_multiprocessing.py
  144. +5 −4 python_modules/dagster-graphql/dagster_graphql_tests/test_python_error.py
  145. +1 −1 python_modules/dagster/dagster/cli/__init__.py
  146. +4 −3 python_modules/dagster/dagster/cli/schedule.py
  147. +1 −1 python_modules/dagster/dagster/core/definitions/dependency.py
  148. +2 −1 python_modules/dagster/dagster/core/definitions/schedule.py
  149. +1 −1 python_modules/dagster/dagster/core/definitions/solid.py
  150. +1 −1 python_modules/dagster/dagster/core/engine/engine_multiprocess.py
  151. +2 −2 python_modules/dagster/dagster/core/events/__init__.py
  152. +5 −6 python_modules/dagster/dagster/core/events/log.py
  153. +1 −1 python_modules/dagster/dagster/core/execution/plan/objects.py
  154. +1 −1 python_modules/dagster/dagster/core/scheduler/__init__.py
  155. +4 −4 python_modules/dagster/dagster/core/scheduler/scheduler.py
  156. +5 −7 python_modules/dagster/dagster/core/scheduler/system_cron_scheduler.py
  157. +2 −2 python_modules/dagster/dagster/core/storage/event_log.py
  158. +2 −2 python_modules/dagster/dagster/core/storage/system_storage.py
  159. +1 −0 python_modules/dagster/dagster/utils/error.py
  160. +2 −2 python_modules/dagster/dagster_tests/core_tests/engine_tests/test_multiprocessing.py
  161. +2 −2 python_modules/dagster/dagster_tests/core_tests/test_event_logging.py
  162. +2 −3 python_modules/dagstermill/dagstermill/examples/repository.py
  163. +3 −2 python_modules/dagstermill/dagstermill/manager.py
  164. +1 −1 python_modules/dagstermill/dagstermill/solids.py
  165. +3 −1 python_modules/dagstermill/dagstermill/translator.py
  166. +1 −1 python_modules/dagstermill/dagstermill_tests/test_basic_dagstermill_solids.py
  167. +1 −2 python_modules/dagstermill/dagstermill_tests/test_cli_commands.py
  168. +1 −1 python_modules/dagstermill/dagstermill_tests/test_context.py
  169. +1 −2 python_modules/dagstermill/dagstermill_tests/test_event_callback.py
  170. +2 −1 python_modules/dagstermill/dagstermill_tests/test_logging.py
  171. +3 −4 python_modules/dagstermill/dagstermill_tests/test_manager.py
  172. +2 −2 python_modules/dagstermill/dagstermill_tests/test_repository.py
  173. +2 −3 python_modules/lakehouse/lakehouse_tests/conftest.py
  174. +1 −1 python_modules/lakehouse/lakehouse_tests/repo.py
  175. +2 −3 python_modules/lakehouse/lakehouse_tests/test_basic_pyspark_lakehouse.py
  176. +2 −2 python_modules/lakehouse/lakehouse_tests/test_basic_sqllite_lakehouse.py
  177. +1 −2 python_modules/lakehouse/lakehouse_tests/test_lakehouse_table.py
  178. +3 −4 python_modules/lakehouse/lakehouse_tests/test_pyspark_custom_url_scheme_lakehouse.py
  179. +5 −6 python_modules/lakehouse/lakehouse_tests/test_typed_pyspark_lakehouse.py
  180. +1 −2 python_modules/libraries/dagster-aws/dagster_aws/cli/cli.py
  181. +1 −3 python_modules/libraries/dagster-aws/dagster_aws/cli/config.py
  182. +1 −1 python_modules/libraries/dagster-aws/dagster_aws/cli/term.py
  183. +2 −3 python_modules/libraries/dagster-aws/dagster_aws/s3/file_cache.py
  184. +1 −2 python_modules/libraries/dagster-aws/dagster_aws/s3/s3_fake_resource.py
  185. +2 −2 python_modules/libraries/dagster-aws/dagster_aws_tests/cli_tests/test_config.py
  186. +0 −2 python_modules/libraries/dagster-aws/dagster_aws_tests/cli_tests/test_term.py
  187. +2 −3 python_modules/libraries/dagster-aws/dagster_aws_tests/cloudwatch_tests/test_loggers.py
  188. +1 −1 python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr.py
  189. +3 −3 python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_file_handle_to_s3.py
  190. +3 −3 python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_intermediate_store.py
  191. +4 −4 python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_s3_file_manager.py
  192. +1 −2 python_modules/libraries/dagster-bash/dagster_bash_tests/test_solids.py
  193. +2 −2 python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_resources.py
  194. +3 −3 python_modules/libraries/dagster-dbt/dagster_dbt/__init__.py
  195. +1 −1 python_modules/libraries/dagster-dbt/dagster_dbt_tests/test_regexes.py
  196. +1 −2 python_modules/libraries/dagster-gcp/dagster_gcp/bigquery/solids.py
  197. +9 −10 python_modules/libraries/dagster-gcp/dagster_gcp_tests/bigquery_tests/test_solids.py
  198. +1 −2 python_modules/libraries/dagster-gcp/dagster_gcp_tests/dataproc_tests/test_resources.py
  199. +2 −3 python_modules/libraries/dagster-ge/dagster_ge_tests/test_pandas_ge.py
  200. +1 −2 python_modules/libraries/dagster-pagerduty/dagster_pagerduty_tests/test_resources.py
  201. +1 −2 python_modules/libraries/dagster-pandas/dagster_pandas/examples/__init__.py
  202. +2 −2 python_modules/libraries/dagster-pandas/dagster_pandas/examples/pandas_hello_world/pipeline.py
  203. +4 −5 ...dules/libraries/dagster-pandas/dagster_pandas_tests/pandas_hello_world/test_pandas_hello_world.py
  204. +1 −2 python_modules/libraries/dagster-pandas/dagster_pandas_tests/test_config_driven_df.py
  205. +2 −2 ...on_modules/libraries/dagster-pandas/dagster_pandas_tests/test_pandas_hello_world_library_slide.py
  206. +1 −2 python_modules/libraries/dagster-pandas/dagster_pandas_tests/test_pandas_metadata.py
  207. +1 −2 python_modules/libraries/dagster-pandas/dagster_pandas_tests/test_pandas_solids.py
  208. +1 −2 python_modules/libraries/dagster-pandas/dagster_pandas_tests/test_pandas_user_error.py
  209. +2 −2 python_modules/libraries/dagster-papertrail/dagster_papertrail_tests/test_loggers.py
  210. +2 −3 python_modules/libraries/dagster-pyspark/dagster_pyspark/__init__.py
  211. +1 −2 python_modules/libraries/dagster-slack/dagster_slack_tests/test_resources.py
  212. +2 −2 python_modules/libraries/dagster-snowflake/dagster_snowflake_tests/test_resources.py
  213. +1 −2 python_modules/libraries/dagster-spark/dagster_spark_tests/test_error.py
  214. +2 −2 python_modules/libraries/dagster-ssh/dagster_ssh_tests/test_resources.py
  215. +2 −2 python_modules/libraries/dagster-ssh/dagster_ssh_tests/test_solids.py
  216. +1 −2 scala_modules/scripts/deploy.py
@@ -1,13 +1,14 @@
import os
import sys

import yaml
from defines import SupportedPython
from step_builder import StepBuilder

SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))

sys.path.append(SCRIPT_PATH)

from step_builder import StepBuilder
from defines import SupportedPython


def deploy_scala():
@@ -26,4 +26,3 @@ def spark_session():

if __name__ == '__main__':
spark_session()

@@ -1,12 +1,13 @@
import os
import sys

import yaml
from defines import SupportedPython
from step_builder import StepBuilder

SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
sys.path.append(SCRIPT_PATH)

from step_builder import StepBuilder
from defines import SupportedPython

if __name__ == "__main__":
steps = [
@@ -2,7 +2,6 @@
import sys

import yaml

from defines import SupportedPython, SupportedPython3s, SupportedPythons
from step_builder import StepBuilder

@@ -362,9 +361,8 @@ def deploy_trigger_step():
.on_medium_instance()
.build(),
StepBuilder("isort")
.run("pip install isort>=4.3.21", "make isort")
.on_integration_image(SupportedPython.V3_7)
.on_medium_instance()
.run("pip install isort>=4.3.21", "make isort", "git diff --exit-code")
.on_python_image(SupportedPython.V3_7)
.build(),
StepBuilder("black")
# black 18.9b0 doesn't support py27-compatible formatting of the below invocation (omitting
@@ -1,11 +1,12 @@
import os
import sys

from defines import SupportedPythons

SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))

sys.path.append(SCRIPT_PATH)

from defines import SupportedPythons

TIMEOUT_IN_MIN = 20

@@ -1,9 +1,9 @@
[settings]

line_length=100
sections=FUTURE,STDLIB,THIRDPARTY,DAGSTER,FIRSTPARTY,LOCALFOLDER
sections=FUTURE,STDLIB,THIRDPARTY,DAGSTER,LOCALFOLDER
known_dagster=dagster
known_first_party=dagster_graphql,dagit,dagster_airflow,dagster_dask,dagstermill,lakehouse,dagster_aws,dagster_bash,dagster_datadog,dagster_gcp,dagster_ge,dagster_pagerduty,dagster_pandas,dagster_papertrail,dagster_pyspark,dagster_slack,dagster_snowflake,dagster_spark,dagster_ssh
default_section = THIRDPARTY
multi_line_output=3
case_sensitive=True
include_trailing_comma=True
@@ -80,4 +80,4 @@ graphql_tests:
pytest examples/dagster_examples_tests/graphql_tests/ python_modules/dagster-graphql/dagster_graphql_tests/graphql/ -s -vv

isort:
cd python_modules && isort -rc .
isort -rc .
@@ -6,10 +6,10 @@
"""
from __future__ import absolute_import

import click
import os
import shutil

import click
from git_tag import get_most_recent_git_tag


@@ -22,7 +22,6 @@
import packaging.version
import slackclient
import virtualenv

from git_tag import get_git_tag, get_most_recent_git_tag, set_git_tag
from pypirc import ConfigFileError, RCParser

@@ -1,9 +1,8 @@
import dagstermill as dm
from dagster_examples.util import download_file

from dagster import DependencyDefinition, InputDefinition, Path, PipelineDefinition
from dagster.utils import script_relative_path
from dagster_examples.util import download_file


k_means_iris_solid = dm.define_dagstermill_solid(
'k_means_iris',
@@ -1,9 +1,8 @@
import dagstermill as dm
from dagster_examples.util import download_file

from dagster import DependencyDefinition, Field, InputDefinition, Int, Path, PipelineDefinition
from dagster.utils import script_relative_path
from dagster_examples.util import download_file


k_means_iris_solid = dm.define_dagstermill_solid(
'k_means_iris',
@@ -1,8 +1,8 @@
from dagster_aws import S3Coordinate

from dagster import EventMetadataEntry, ExpectationResult, Field, FileHandle, Output, solid
from dagster.utils.temp_file import get_temp_file_name

from dagster_aws import S3Coordinate


@solid(
config={
@@ -1,16 +1,14 @@
# pylint: disable=no-value-for-parameter
"""Pipeline definitions for the airline_demo."""
from dagster import ModeDefinition, PresetDefinition, composite_solid, file_relative_path, pipeline

from dagster.core.storage.file_cache import fs_file_cache
from dagster.core.storage.temp_file_manager import tempfile_resource

from dagster_aws.s3.resources import s3_resource
from dagster_aws.s3.file_cache import s3_file_cache
from dagster_aws.s3.file_manager import S3FileHandle
from dagster_aws.s3.system_storage import s3_plus_default_storage_defs
from dagster_aws.s3.resources import s3_resource
from dagster_aws.s3.solids import file_handle_to_s3
from dagster_aws.s3.system_storage import s3_plus_default_storage_defs

from dagster import ModeDefinition, PresetDefinition, composite_solid, file_relative_path, pipeline
from dagster.core.storage.file_cache import fs_file_cache
from dagster.core.storage.temp_file_manager import tempfile_resource

from .cache_file_from_s3 import cache_file_from_s3
from .resources import postgres_db_info_resource, redshift_db_info_resource, spark_session_local
@@ -21,18 +19,17 @@
delays_vs_fares_nb,
eastbound_delays,
ingest_csv_file_handle_to_spark,
load_data_to_database_from_spark,
join_q2_data,
load_data_to_database_from_spark,
process_sfo_weather_data,
q2_sfo_outbound_flights,
s3_to_df,
s3_to_dw_table,
sfo_delays_by_destination,
tickets_with_destination,
westbound_delays,
s3_to_dw_table,
s3_to_df,
)


test_mode = ModeDefinition(
name='test',
resource_defs={
@@ -1,6 +1,7 @@
from pyspark.sql import SparkSession

from dagster import resource, Field
from dagster import Field, resource

from .types import DbInfo
from .utils import (
create_postgres_db_url,
@@ -3,6 +3,9 @@
import os
import re

from dagster_aws.s3.solids import S3Coordinate
from dagster_pyspark import DataFrame
from dagstermill import define_dagstermill_solid
from sqlalchemy import text

from dagster import (
@@ -14,22 +17,18 @@
InputDefinition,
Int,
Materialization,
OutputDefinition,
Output,
OutputDefinition,
String,
check,
composite_solid,
solid,
)
from dagster_aws.s3.solids import S3Coordinate
from dagster_pyspark import DataFrame
from dagstermill import define_dagstermill_solid

from .types import SqlTableName
from .cache_file_from_s3 import cache_file_from_s3
from .types import SqlTableName
from .unzip_file_handle import unzip_file_handle


PARQUET_SPECIAL_CHARACTERS = r'[ ,;{}()\n\t=]'


@@ -1,6 +1,6 @@
import zipfile

from dagster import solid, FileHandle, String
from dagster import FileHandle, String, solid


@solid(
@@ -6,14 +6,14 @@
and this should serve as a reasonable template to build upon.
'''
from dagster import (
SolidInvocation,
PipelineDefinition,
DependencyDefinition,
lambda_solid,
file_relative_path,
check,
InputDefinition,
OutputDefinition,
PipelineDefinition,
SolidInvocation,
check,
file_relative_path,
lambda_solid,
)
from dagster.utils import load_yaml_from_path

@@ -5,6 +5,10 @@
import shutil

import six
from dagster_aws.s3.resources import s3_resource
from dagster_aws.s3.utils import S3Logger
from dagster_snowflake import snowflake_resource
from dagster_spark import SparkSolidDefinition

from dagster import (
Bool,
@@ -25,12 +29,7 @@
pipeline,
solid,
)
from dagster.utils import safe_isfile, mkdir_p

from dagster_aws.s3.resources import s3_resource
from dagster_aws.s3.utils import S3Logger
from dagster_snowflake import snowflake_resource
from dagster_spark import SparkSolidDefinition
from dagster.utils import mkdir_p, safe_isfile


class FileExistsAtPath(RuntimeType):
@@ -3,24 +3,24 @@
import datetime
import os

from dagster_gcp import (
bigquery_resource,
bq_solid_for_queries,
dataproc_resource,
dataproc_solid,
import_gcs_paths_to_bq,
)

from dagster import (
composite_solid,
file_relative_path,
pipeline,
solid,
Field,
List,
ModeDefinition,
PresetDefinition,
String,
)

from dagster_gcp import (
bigquery_resource,
import_gcs_paths_to_bq,
bq_solid_for_queries,
dataproc_resource,
dataproc_solid,
composite_solid,
file_relative_path,
pipeline,
solid,
)

PROJECT_ID = os.getenv('GCP_PROJECT_ID')
@@ -1,19 +1,18 @@
import datetime
import os

from dagster_gcp import bigquery_resource, dataproc_resource
from google.cloud.bigquery.job import LoadJobConfig, QueryJobConfig

from dagster import (
file_relative_path,
pipeline,
solid,
InputDefinition,
ModeDefinition,
Nothing,
PresetDefinition,
file_relative_path,
pipeline,
solid,
)
from dagster_gcp import bigquery_resource, dataproc_resource


PROJECT_ID = os.getenv('GCP_PROJECT_ID')
DEPLOY_BUCKET_PREFIX = os.getenv('GCP_DEPLOY_BUCKET_PREFIX')
@@ -1,12 +1,11 @@
import datetime
import os

from google.cloud.bigquery.job import LoadJobConfig, QueryJobConfig

from dagster import solid, pipeline, InputDefinition, Nothing
from dagster_gcp.bigquery.resources import BigQueryClient
from dagster_gcp.dataproc.resources import DataprocResource
from google.cloud.bigquery.job import LoadJobConfig, QueryJobConfig

from dagster import InputDefinition, Nothing, pipeline, solid

PROJECT_ID = os.getenv('GCP_PROJECT_ID')
DEPLOY_BUCKET_PREFIX = os.getenv('GCP_DEPLOY_BUCKET_PREFIX')
@@ -1,7 +1,7 @@
# pylint: disable=no-value-for-parameter
# encoding: utf-8

from dagster import execute_pipeline, solid, Field, pipeline
from dagster import Field, execute_pipeline, pipeline, solid


@solid(config={'language': Field(str, is_optional=True, default_value='en-us')})
@@ -2,7 +2,7 @@

import collections

from dagster import Field, Int, lambda_solid, solid, pipeline, as_dagster_type
from dagster import Field, Int, as_dagster_type, lambda_solid, pipeline, solid

Counter = as_dagster_type(collections.Counter)

@@ -2,7 +2,7 @@

import collections

from dagster import Any, Field, lambda_solid, solid, pipeline, as_dagster_type
from dagster import Any, Field, as_dagster_type, lambda_solid, pipeline, solid

Counter = as_dagster_type(collections.Counter)

@@ -1,4 +1,4 @@
from dagster import solid, pipeline, ExpectationResult, Output
from dagster import ExpectationResult, Output, pipeline, solid


@solid
@@ -1,4 +1,4 @@
from dagster import pipeline, lambda_solid
from dagster import lambda_solid, pipeline


@lambda_solid

0 comments on commit 49ec982

Please sign in to comment.
You can’t perform that action at this time.