Skip to content
Permalink
Browse files

Pre-load jar deps for scala_modules in integration image

Summary: This will stash a bunch of jars in `/root/.ivy2` in the integration image

Test Plan: n/a

Reviewers: schrockn, alangenfeld

Reviewed By: schrockn

Differential Revision: https://dagster.phacility.com/D416
  • Loading branch information...
natekupp committed Jun 17, 2019
1 parent b821779 commit 3226129371cb94d1af5c11931e7a76880f7d9499
@@ -1,18 +1,27 @@
#! /bin/bash

set -eu
set -eux

ROOT=$(git rev-parse --show-toplevel)
pushd $ROOT/.buildkite/images/integration/

docker build . --build-arg PYTHON_VERSION=3.7.3 --build-arg PYTHON_MAJOR_VERSION=3 -t dagster/buildkite-integration:py3.7.3-v3
docker push dagster/buildkite-integration:py3.7.3-v3
function cleanup {
rm -rf scala_modules
}

docker build . --build-arg PYTHON_VERSION=3.6.8 --build-arg PYTHON_MAJOR_VERSION=3 -t dagster/buildkite-integration:py3.6.8-v3
docker push dagster/buildkite-integration:py3.6.8-v3
# ensure cleanup happens on error or normal exit
trap cleanup EXIT

docker build . --build-arg PYTHON_VERSION=3.5.7 --build-arg PYTHON_MAJOR_VERSION=3 -t dagster/buildkite-integration:py3.5.7-v3
docker push dagster/buildkite-integration:py3.5.7-v3
cp -R $ROOT/scala_modules .

docker build . --build-arg PYTHON_VERSION=2.7.16 --build-arg PYTHON_MAJOR_VERSION=2 -t dagster/buildkite-integration:py2.7.16-v3
docker push dagster/buildkite-integration:py2.7.16-v3
docker build . --build-arg PYTHON_VERSION=3.7.3 --build-arg PYTHON_MAJOR_VERSION=3 -t dagster/buildkite-integration:py3.7.3-v4
docker push dagster/buildkite-integration:py3.7.3-v4

docker build . --build-arg PYTHON_VERSION=3.6.8 --build-arg PYTHON_MAJOR_VERSION=3 -t dagster/buildkite-integration:py3.6.8-v4
docker push dagster/buildkite-integration:py3.6.8-v4

docker build . --build-arg PYTHON_VERSION=3.5.7 --build-arg PYTHON_MAJOR_VERSION=3 -t dagster/buildkite-integration:py3.5.7-v4
docker push dagster/buildkite-integration:py3.5.7-v4

docker build . --build-arg PYTHON_VERSION=2.7.16 --build-arg PYTHON_MAJOR_VERSION=2 -t dagster/buildkite-integration:py2.7.16-v4
docker push dagster/buildkite-integration:py2.7.16-v4
@@ -81,7 +81,7 @@ RUN set -ex; \
fi; \
apt-get update; \
apt-get install -y \
openjdk-8-jre-headless="$JAVA_DEBIAN_VERSION"; \
openjdk-8-jre-headless="$JAVA_DEBIAN_VERSION"; \
apt-get install -y \
openjdk-8-jdk-headless="$JAVA_DEBIAN_VERSION" \
; \
@@ -121,6 +121,10 @@ RUN pip --no-cache-dir install pyspark==2.4.0
ADD trigger_maven.py .
RUN python trigger_maven.py

# Pre-load jars for scala_modules by running a compile
ADD scala_modules scala_modules
RUN cd scala_modules && make compile

# Postgres (libpq) required for the airline demo
RUN pip install tox
RUN apt-get update && apt-get -y install libpq-dev
@@ -132,5 +136,3 @@ ARG PYTHON_MAJOR_VERSION
ADD snapshot-reqs-$PYTHON_MAJOR_VERSION.txt /tmp/snapshot-reqs.txt
RUN pip install --upgrade pip;\
pip install -r /tmp/snapshot-reqs.txt


@@ -39,7 +39,7 @@ def on_integration_image(self, ver, env=None):
settings = self._base_docker_settings()

# version like dagster/buildkite-integration:py3.7.3-v3
settings["image"] = "dagster/buildkite-integration:py" + ver + '-v3'
settings["image"] = "dagster/buildkite-integration:py" + ver + '-v4'

# map the docker socket to enable docker to be run from inside docker
settings["volumes"] = ["/var/run/docker.sock:/var/run/docker.sock"]
@@ -1,12 +1,12 @@
# Dockerfile for Airflow

FROM dagster/buildkite-integration:py3.7.3-v3
FROM dagster/buildkite-integration:py3.7.3-v4

RUN mkdir -p /tmp/results

WORKDIR /tmp/

ADD . .
ADD . .


# Install all local modules at once with no deps to
@@ -1,4 +1,4 @@
FROM dagster/buildkite-integration:py3.7.3-v3
FROM dagster/buildkite-integration:py3.7.3-v4

ADD dagster dagster
ADD dagster-graphql dagster-graphql
@@ -6,6 +6,9 @@ style:
sbt scalafmtCheck


compile:
sbt events/compile

all:
sbt events/assembly

0 comments on commit 3226129

Please sign in to comment.
You can’t perform that action at this time.