Permalink
Browse files

cleaned up, renamed, combined back with source.ml

  • Loading branch information...
cfregly committed Aug 12, 2017
1 parent aecd8b3 commit 6261c4f31105e40ab8b24ccc7834f9181f4e5aaf
Showing 10,203 changed files with 63,325 additions and 880,509 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
File renamed without changes.
File renamed without changes.
File renamed without changes.
View
@@ -0,0 +1,72 @@
FROM fluxcapacitor/package-spark-2.1.0:master
WORKDIR /root
ENV \
AIRFLOW_HOME=/root/airflow
# MySql Python Adapter (Used by SQLAlchemy/Airflow)
RUN \
apt-get update \
&& apt-get install -y python-mysqldb \
&& apt-get install -y mysql-client \
&& apt-get install -y libmysql-java \
&& apt-get install -y libmysqlclient-dev
RUN \
pip install mysqlclient==1.3.10
#RUN \
# echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list \
# && gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9 \
# && gpg -a --export E084DAB9 | apt-key add - \
# && apt-get update \
# && apt-get install -y r-base \
# && apt-get install -y r-base-dev
RUN \
conda install --yes -c conda-forge gitpython=2.1.1
# This must match the version of Docker on the HOST machine that launces this Docker image.
# In a Cloud-based Kubernetes environment, this is whatever version is burned into the image (ie. AMI on EC2)
# We're currently using 293135079892/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-11-16 on AWS.
# This is pinned to Docker 1.11
RUN \
curl -fsSLO https://get.docker.com/builds/Linux/x86_64/docker-1.11.2.tgz \
&& tar --strip-components=1 -xvzf docker-1.11.2.tgz -C /usr/local/bin
ENV \
KUBERNETES_VERSION=1.6.2
RUN \
wget https://storage.googleapis.com/kubernetes-release/release/v$KUBERNETES_VERSION/bin/linux/amd64/kubectl
RUN \
chmod a+x kubectl \
&& mv kubectl /usr/local/bin/kubectl
RUN \
mkdir -p $AIRFLOW_HOME \
&& cd $AIRFLOW_HOME
RUN \
pip install airflow==1.8.0 \
&& pip install celery[redis]==4.0.2 \
&& pip install flower==0.9.1
RUN \
pip install tensorflow==1.0.1
# Temp workaround for running Celery worker as root
ENV \
C_FORCE_ROOT=true
COPY config/airflow/airflow.cfg $AIRFLOW_HOME/airflow.cfg
COPY run run
COPY airflow/ airflow/
EXPOSE 8080 5000 5555 8793
CMD ["supervise", "."]
@@ -0,0 +1,48 @@
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: scheduler-airflow
labels:
app: scheduler-airflow
spec:
replicas: 1
selector:
matchLabels:
app: scheduler-airflow
template:
metadata:
labels:
app: scheduler-airflow
spec:
containers:
- name: scheduler-airflow
image: docker.io/fluxcapacitor/scheduler-airflow:master
resources:
limits:
cpu: 1000m
memory: 2000Mi
requests:
cpu: 100m
memory: 200Mi
imagePullPolicy: "Always"
env:
- name: GET_HOSTS_FROM
value: env
- name: SPARK_MASTER
value: spark://apachespark-master-2-1-0:7077
- name: SPARK_SUBMIT_ARGS
value: ""
ports:
- containerPort: 8080
- containerPort: 5000
- containerPort: 5555
- containerPort: 8793
volumeMounts:
- name: dockersocket
mountPath: /var/run/docker.sock
securityContext:
privileged: true
volumes:
- hostPath:
path: /var/run/docker.sock
name: dockersocket
@@ -0,0 +1,26 @@
apiVersion: v1
kind: Service
metadata:
name: scheduler-airflow
labels:
app: scheduler-airflow
# dns: route53
# annotations:
# domainName: airflow.your.domain.com
spec:
type: LoadBalancer
ports:
- name: airflow-webui
port: 80
targetPort: 8080
- name: github-webhook
port: 5000
targetPort: 5000
- name: celery-webui
port: 5555
targetPort: 5555
- name: airflow-logs
port: 8793
targetPort: 8793
selector:
app: scheduler-airflow
View
@@ -0,0 +1,19 @@
#!/bin/bash
/usr/local/bin/dockerd
mysql --host=sql-mysql-master --port=3306 --user=root --password=password -e "CREATE DATABASE IF NOT EXISTS airflow"
sleep 2
# TODO: Don't re-init upon process restart
airflow initdb
sleep 2
# TODO: Separate these out into separate containers - perhaps in same pod
airflow flower &
airflow webserver &
airflow worker &
airflow scheduler
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,35 @@
apiVersion: extensions/v1beta
kind: Deployment
metadata:
name: cassandra
labels:
app: cassandra
spec:
replicas: 1
selector:
matchLabels:
app: cassandra
template:
metadata:
labels:
app: cassandra
spec:
containers:
- name: cassandra
image: docker.io/fluxcapacitor/cassandra:master
resources:
limits:
cpu: 1000m
memory: 4000Mi
requests:
cpu: 100m
memory: 400Mi
imagePullPolicy: "Always"
env:
- name: GET_HOSTS_FROM
value: env
ports:
- containerPort: 7199
- containerPort: 9042
- containerPort: 9160
- containerPort: 7000
@@ -0,0 +1,23 @@
apiVersion: v1
kind: Service
metadata:
name: cassandra
labels:
app: cassandra
spec:
type: NodePort
ports:
- name: jmx
port: 7199
targetPort: 7199
- name: native
port: 9042
targetPort: 9042
- name: thrift
port: 9160
targetPort: 9160
- name: gossip
port: 7000
targetPort: 7000
selector:
app: cassandra
File renamed without changes.
View

This file was deleted.

Oops, something went wrong.
View

This file was deleted.

Oops, something went wrong.
View

This file was deleted.

Oops, something went wrong.
View

This file was deleted.

Oops, something went wrong.
View

This file was deleted.

Oops, something went wrong.
View

This file was deleted.

Oops, something went wrong.
Oops, something went wrong.

0 comments on commit 6261c4f

Please sign in to comment.