From 784a89801b42787ea79934339a3503b04d399a9c Mon Sep 17 00:00:00 2001 From: alangenfeld Date: Thu, 18 Apr 2019 08:49:37 -0700 Subject: [PATCH] buildkite setup --- .buildkite/pipeline.py | 149 ++++++++++++++++++ .circleci/config.yml | 3 +- Makefile | 26 ++- .../dagster/core/system_config/objects.py | 1 - 4 files changed, 175 insertions(+), 4 deletions(-) create mode 100644 .buildkite/pipeline.py diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py new file mode 100644 index 0000000000000..d9fb494283008 --- /dev/null +++ b/.buildkite/pipeline.py @@ -0,0 +1,149 @@ +import yaml + +DOCKER_PLUGIN = "docker#v3.1.0" + + +# This should be an enum once we make our own buildkite AMI with py3 +class SupportedPython: + V3_7 = "3.7" + V3_6 = "3.6" + V3_5 = "3.5" + V2_7 = "2.7" + + +SupportedPythons = [ + SupportedPython.V3_7, + SupportedPython.V3_6, + SupportedPython.V3_5, + SupportedPython.V2_7, +] + +IMAGE_MAP = { + SupportedPython.V3_7: "python:3.7", + SupportedPython.V3_6: "python:3.6", + SupportedPython.V3_5: "python:3.5", + SupportedPython.V2_7: "python:2.7", +} + +TOX_MAP = { + SupportedPython.V3_7: "py37", + SupportedPython.V3_6: "py36", + SupportedPython.V3_5: "py35", + SupportedPython.V2_7: "py27", +} + + +class StepBuilder: + def __init__(self, label): + self._step = {"label": label} + + def run(self, *argc): + self._step["command"] = "\n".join(argc) + return self + + def onImage(self, img): + if img in IMAGE_MAP: + img = IMAGE_MAP[img] + self._step["plugins"] = [{DOCKER_PLUGIN: {"image": img}}] + return self + + def withEnvironVar(self, var): + self._step["env"] = {var: "${{{}?}}".format(var)} + return self + + def build(self): + return self._step + + +def wait_step(): + return "wait" + + +def python_modules_tox_tests(directory, prereqs=None): + label = directory.replace("/", "-") + tests = [] + for version in SupportedPythons: + coverage = ".coverage.{}.{}.$BUILDKITE_BUILD_ID".format(label, version) + tox_command = [] + if prereqs: + tox_command += prereqs + tox_command += [ + "pip install tox;", + "cd python_modules/{}".format(directory), + "tox -e {}".format(TOX_MAP[version]), + "mv .coverage {}".format(coverage), + "buildkite-agent artifact upload {}".format(coverage), + ] + tests.append( + StepBuilder("{} tests ({})".format(directory, version)) + .run(*tox_command) + .onImage(version) + .build() + ) + + return tests + + +if __name__ == "__main__": + steps = [ + StepBuilder("pylint") + .run("make dev_install", "make pylint") + .onImage(SupportedPython.V3_7) + .build(), + StepBuilder("black") + .run("pip install black==18.9b0", "make check_black") + .onImage(SupportedPython.V3_7) + .build(), + StepBuilder("docs snapshot test") + .run( + "pip install -r python_modules/dagster/dev-requirements.txt -qqq", + "pip install -e python_modules/dagster -qqq", + "pytest -vv python_modules/dagster/docs", + ) + .onImage(SupportedPython.V3_7) + .build(), + StepBuilder("dagit webapp tests") + .run( + "pip install -r python_modules/dagster/dev-requirements.txt -qqq", + "pip install -e python_modules/dagster -qqq", + "pip install -e python_modules/dagster-graphql -qqq", + "pip install -e python_modules/dagit -qqq", + "pip install -r python_modules/dagit/dev-requirements.txt -qqq", + "cd js_modules/dagit", + "yarn install --offline", + "yarn run ts", + "yarn run jest", + "yarn run check-prettier", + "yarn generate-types", + "git diff --exit-code", + ) + .onImage("nikolaik/python-nodejs:python3.7-nodejs11") + .build(), + ] + steps += python_modules_tox_tests("dagster") + steps += python_modules_tox_tests("dagit", ["apt-get update", "apt-get install -y xdg-utils"]) + steps += python_modules_tox_tests("dagster-graphql") + steps += python_modules_tox_tests("dagstermill") + steps += python_modules_tox_tests("libraries/dagster-pandas") + steps += python_modules_tox_tests("libraries/dagster-ge") + steps += python_modules_tox_tests("libraries/dagster-aws") + steps += python_modules_tox_tests("libraries/dagster-snowflake") + steps += python_modules_tox_tests("libraries/dagster-spark") + + steps += [ + wait_step(), # wait for all previous steps to finish + StepBuilder("coverage") + .run( + "pip install coverage coveralls", + "mkdir -p tmp", + 'buildkite-agent artifact download ".coverage*" tmp/', + "cd tmp", + "coverage combine", + "coveralls", + ) + .withEnvironVar('COVERALLS_REPO_TOKEN') + .onImage(SupportedPython.V3_7) + .build(), + ] + + print(yaml.dump({"steps": steps}, default_flow_style=False, default_style="|")) diff --git a/.circleci/config.yml b/.circleci/config.yml index 66fd77fe40582..26921bfd25119 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,7 +60,6 @@ workflows: - library-dagster-pandas-py27: context: coveralls - - library-dagster-ge-py37: context: coveralls - library-dagster-ge-py36: @@ -253,7 +252,7 @@ jobs: - run: name: Lint command: | - make pylint + make pylint-iterative docs-snapshot: docker: diff --git a/Makefile b/Makefile index 727bc017ccf28..05a5d5ee3ebe5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # This is a hack because we are getting timeouts on CircleCI running pylint on all the targets # at once -pylint: +pylint-iterative: set -e; for target in `cat .pylint_targets` ; do \ echo $$target; \ @@ -8,8 +8,32 @@ pylint: done; set +e; +pylint: + pylint -j 0 `cat .pylint_targets` --rcfile=.pylintrc --disable=R,C + update_doc_snapshot: pytest python_modules/dagster/docs --snapshot-update black: black python_modules --line-length 100 -S --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/" -N + +check_black: + black python_modules --check --line-length 100 -S --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/" -N + +dev_install: + pip install -r python_modules/dagster/dev-requirements.txt -qqq + pip install -e python_modules/dagster -qqq + pip install -e python_modules/dagster-graphql -qqq + pip install -e python_modules/dagit -qqq + pip install -r python_modules/dagit/dev-requirements.txt -qqq + pip install -e python_modules/dagstermill -qqq + SLUGIFY_USES_TEXT_UNIDECODE=yes pip install -e python_modules/dagster-airflow -qqq + pip install -e python_modules/libraries/dagster-aws -qqq + pip install -e python_modules/libraries/dagster-ge -qqq + pip install -e python_modules/libraries/dagster-pandas -qqq + pip install -e python_modules/libraries/dagster-snowflake -qqq + pip install -e python_modules/libraries/dagster-spark -qqq + pip install -e python_modules/libraries/dagster-pyspark -qqq + pip install -e python_modules/automation -qqq + pip install -e examples/event-pipeline-demo -qqq + pip install -e examples/airline-demo -qqq \ No newline at end of file diff --git a/python_modules/dagster/dagster/core/system_config/objects.py b/python_modules/dagster/dagster/core/system_config/objects.py index 4ee70a24bd1dd..c23b108829472 100644 --- a/python_modules/dagster/dagster/core/system_config/objects.py +++ b/python_modules/dagster/dagster/core/system_config/objects.py @@ -110,4 +110,3 @@ def construct_run_storage(self): raise DagsterInvariantViolationError( 'Invalid storage specified {}'.format(self.storage_mode) ) -