Skip to content
Browse files

buildkite setup (#1286)

Test Plan: pipeline works in buildkite
  • Loading branch information...
alangenfeld committed Apr 23, 2019
1 parent 809ceb4 commit 854c92ee19d3ee12c53c043271879c9cfe7b4f9a
Showing with 184 additions and 2 deletions.
  1. +3 −0 .arcconfig
  2. +159 −0 .buildkite/
  3. +22 −1 Makefile
  4. +0 −1 python_modules/dagster/dagster/core/system_config/
@@ -0,0 +1,3 @@
"phabricator.uri" : ""
@@ -0,0 +1,159 @@
import yaml

DOCKER_PLUGIN = "docker#v3.1.0"

# This should be an enum once we make our own buildkite AMI with py3
class SupportedPython:
V3_7 = "3.7"
V3_6 = "3.6"
V3_5 = "3.5"
V2_7 = "2.7"

SupportedPythons = [

SupportedPython.V3_7: "python:3.7",
SupportedPython.V3_6: "python:3.6",
SupportedPython.V3_5: "python:3.5",
SupportedPython.V2_7: "python:2.7",

SupportedPython.V3_7: "py37",
SupportedPython.V3_6: "py36",
SupportedPython.V3_5: "py35",
SupportedPython.V2_7: "py27",

class StepBuilder:
def __init__(self, label):
self._step = {"label": label}

def run(self, *argc):
self._step["command"] = "\n".join(argc)
return self

def on_docker_image(self, img, env=None):
if img in IMAGE_MAP:
img = IMAGE_MAP[img]

docker = {"image": img}
if env:
docker['environment'] = env

self._step["plugins"] = [{DOCKER_PLUGIN: docker}]
return self

def build(self):
return self._step

def wait_step():
return "wait"

def python_modules_tox_tests(directory, prereqs=None):
label = directory.replace("/", "-")
tests = []
for version in SupportedPythons:
coverage = ".coverage.{label}.{version}.$BUILDKITE_BUILD_ID".format(
label=label, version=version
tox_command = []
if prereqs:
tox_command += prereqs
tox_command += [
"pip install tox;",
"cd python_modules/{directory}".format(directory=directory),
"tox -e {ver}".format(ver=TOX_MAP[version]),
"mv .coverage {file}".format(file=coverage),
"buildkite-agent artifact upload {file}".format(file=coverage),
StepBuilder("{label} tests ({ver})".format(label=label, ver=TOX_MAP[version]))

return tests

if __name__ == "__main__":
steps = [
.run("make dev_install", "make pylint")
# black 18.9b0 doesn't support py27-compatible formatting of the below invocation (omitting
# the trailing comma after **check.opt_dict_param...) -- black 19.3b0 supports multiple python
# versions, but currently doesn't know what to do with from __future__ import print_function --
# see
.run("pip install black==18.9b0", "make check_black")
StepBuilder("docs snapshot test")
"pip install -r python_modules/dagster/dev-requirements.txt -qqq",
"pip install -e python_modules/dagster -qqq",
"pytest -vv python_modules/dagster/docs",
StepBuilder("dagit webapp tests")
"pip install -r python_modules/dagster/dev-requirements.txt -qqq",
"pip install -e python_modules/dagster -qqq",
"pip install -e python_modules/dagster-graphql -qqq",
"pip install -e python_modules/dagit -qqq",
"pip install -r python_modules/dagit/dev-requirements.txt -qqq",
"cd js_modules/dagit",
"yarn install --offline",
"yarn run ts",
"yarn run jest",
"yarn run check-prettier",
"yarn generate-types",
"git diff --exit-code",
steps += python_modules_tox_tests("dagster")
steps += python_modules_tox_tests("dagit", ["apt-get update", "apt-get install -y xdg-utils"])
steps += python_modules_tox_tests("dagster-graphql")
steps += python_modules_tox_tests("dagstermill")
steps += python_modules_tox_tests("libraries/dagster-pandas")
steps += python_modules_tox_tests("libraries/dagster-ge")
steps += python_modules_tox_tests("libraries/dagster-aws")
steps += python_modules_tox_tests("libraries/dagster-snowflake")
steps += python_modules_tox_tests("libraries/dagster-spark")

steps += [
wait_step(), # wait for all previous steps to finish
"pip install coverage coveralls",
"mkdir -p tmp",
'buildkite-agent artifact download ".coverage*" tmp/',
"cd tmp",
"coverage combine",
# COVERALLS_REPO_TOKEN exported by /env in ManagedSecretsBucket

print(yaml.dump({"steps": steps}, default_flow_style=False, default_style="|"))
@@ -1,6 +1,6 @@
# This is a hack because we are getting timeouts on CircleCI running pylint on all the targets
# at once
set -e;
for target in `cat .pylint_targets` ; do \
echo $$target; \
@@ -16,3 +16,24 @@ update_doc_snapshot:

black python_modules --line-length 100 -S --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/" -N

black python_modules --check --line-length 100 -S --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/" -N

pip install -r python_modules/dagster/dev-requirements.txt -qqq
pip install -e python_modules/dagster -qqq
pip install -e python_modules/dagster-graphql -qqq
pip install -e python_modules/dagit -qqq
pip install -r python_modules/dagit/dev-requirements.txt -qqq
pip install -e python_modules/dagstermill -qqq
SLUGIFY_USES_TEXT_UNIDECODE=yes pip install -e python_modules/dagster-airflow -qqq
pip install -e python_modules/libraries/dagster-aws -qqq
pip install -e python_modules/libraries/dagster-ge -qqq
pip install -e python_modules/libraries/dagster-pandas -qqq
pip install -e python_modules/libraries/dagster-snowflake -qqq
pip install -e python_modules/libraries/dagster-spark -qqq
pip install -e python_modules/libraries/dagster-pyspark -qqq
pip install -e python_modules/automation -qqq
pip install -e examples/event-pipeline-demo -qqq
pip install -e examples/airline-demo -qqq
@@ -110,4 +110,3 @@ def construct_run_storage(self):
raise DagsterInvariantViolationError(
'Invalid storage specified {}'.format(self.storage_mode)

0 comments on commit 854c92e

Please sign in to comment.
You can’t perform that action at this time.