Skip to content
Permalink
Browse files
[GOBBLIN-1357] Updates the cluster to use up-to-date images and non-d…
…eprecated ingress

Fixes bug where standalone could not pick up jobs due to env variable not being set

Closes #3196 from Will-Lo/update-k8s-gaas
  • Loading branch information
Will-Lo authored and suvasude committed Jan 22, 2021
1 parent 5ecf9a1 commit f4c75139bf38b73b3fc052cc5c2010cdca9bc4c0
Showing 11 changed files with 85 additions and 32 deletions.
@@ -16,7 +16,7 @@
#

# Gobblin-As-Service configuration properties
gobblin.service.work.dir=/tmp/gobblin-as-service
gobblin.service.work.dir=/etc/gobblin-as-service
fs.uri="file:///"

# Topology Catalog and Store
@@ -0,0 +1,34 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

version: '3'
services:
gobblin-service:
image: apache/gobblin:latest
command: --mode "gobblin-as-service"
volumes:
- "${LOCAL_DATAPACK_DIR}:/etc/templateCatalog"
- "${LOCAL_JOB_DIR}:/etc/gobblin-as-service/jobs"
ports:
- 6956:6956
gobblin-standalone:
image: apache/gobblin:latest
command: --mode "standalone"
volumes:
- "${LOCAL_JOB_DIR}:/etc/gobblin-standalone/jobs"
environment:
- GOBBLIN_JOB_CONFIG_DIR=/etc/gobblin-standalone/jobs
@@ -18,10 +18,10 @@
version: '3'
services:
gobblin-standalone:
image: apache/gobblin:master
image: apache/gobblin:latest
command: --mode "standalone"
volumes:
- "${LOCAL_JOB_DIR}:/tmp/gobblin-standalone/jobs"
- "${LOCAL_JOB_DIR}:/etc/gobblin-standalone/jobs"
zookeeper:
image: wurstmeister/zookeeper
ports:
@@ -30,10 +30,11 @@ COPY ./gobblin-docker/gobblin/alpine-gobblin-latest/entrypoint.sh ./bin/entrypoi
COPY --from=build /home/gobblin/gobblin-dist .
RUN apk add --no-cache bash && \
mkdir /tmp/gobblin && \
mkdir /tmp/gobblin/jobs && \
mkdir /etc/gobblin && \
mkdir /etc/gobblin/jobs && \
chmod +x ./bin/entrypoint.sh

ENV GOBBLIN_WORK_DIR=/tmp/gobblin/
ENV GOBBLIN_JOB_CONFIG_DIR=/tmp/gobblin/jobs
ENV GOBBLIN_JOB_CONFIG_DIR=/etc/gobblin/jobs

ENTRYPOINT ["./bin/entrypoint.sh"]
@@ -30,18 +30,17 @@ GOBBLIN_HOME="$(cd `dirname $0`/..; pwd)"

EXECUTION_MODE=''
CONF_PATH=''
ARGS="$@"
JVM_OPTS=''
ARGS=("$@")

shopt -s nocasematch
for i in $ARGS
for i in "${!ARGS[@]}"
do
case "$1" in
case "${ARGS[$i]}" in
--mode )
EXECUTION_MODE="$2"
shift
EXECUTION_MODE="${ARGS[$i+1]}"
;;
esac
shift
done

if [[ -z "$EXECUTION_MODE" ]]; then
@@ -50,4 +49,4 @@ if [[ -z "$EXECUTION_MODE" ]]; then
exit 1
fi

./bin/gobblin.sh service $EXECUTION_MODE start --log-to-stdout $ARGS
./bin/gobblin.sh service $EXECUTION_MODE start --log-to-stdout "$@"
@@ -44,7 +44,7 @@ Run these commands to start the docker image:

`docker pull apache/gobblin:latest`

`docker run -v $LOCAL_JOB_DIR:/tmp/gobblin-standalone/jobs apache/gobblin:latest --mode standalone`
`docker run -v $LOCAL_JOB_DIR:/etc/gobblin-standalone/jobs apache/gobblin:latest --mode standalone`

After the container spins up, put the [wikipedia.pull](https://github.com/apache/incubator-gobblin/blob/master/gobblin-example/src/main/resources/wikipedia.pull) in ${LOCAL_JOB_DIR}. You will see the Gobblin daemon pick up the job, and the result output is in ${LOCAL_JOB_DIR}/job-output/.

@@ -98,7 +98,7 @@ Similar to standalone working directory settings:

Run these commands to start the docker image:

`docker run -p 6956:6956 -v $GAAS_JOB_DIR:/tmp/gobblin-as-service/jobs -v $LOCAL_DATAPACK_DIR:/tmp/templateCatalog apache/gobblin --mode gobblin-as-service`
`docker run -p 6956:6956 -v $GAAS_JOB_DIR:/etc/gobblin-as-service/jobs -v $LOCAL_DATAPACK_DIR:/etc/templateCatalog apache/gobblin --mode gobblin-as-service`

The GaaS will be started, and the service can now be accessed on localhost:6956.

@@ -27,12 +27,14 @@ spec:
name: flowconfig-templates
containers:
- name: gobblin-service
image: will97/gobblin-as-a-service:latest
image: apache/gobblin:latest
args: ["--mode", "gobblin-as-service", "--jvmopts", "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap"]
volumeMounts:
- name: shared-jobs
mountPath: /tmp/gobblin-as-service/jobs
mountPath: /etc/gobblin-as-service/jobs
- name: flowconfig-templates
mountPath: /tmp/templateCatalog
mountPath: /etc/templateCatalog
imagePullPolicy: Never # TODO: Remove this once docker images are deployed to Apache DockerHub post-graduation
---
apiVersion: apps/v1
kind: Deployment
@@ -57,7 +59,12 @@ spec:
claimName: shared-jobs-claim
containers:
- name: gobblin-standalone
image: will97/gobblin-standalone:latest
image: apache/gobblin:latest
args: ["--mode", "standalone", "--jvmopts", "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap"]
volumeMounts:
- name: shared-jobs
mountPath: /tmp/gobblin-standalone/jobs
mountPath: /etc/gobblin-standalone/jobs
env:
- name: GOBBLIN_JOB_CONFIG_DIR
value: /etc/gobblin-standalone/jobs
imagePullPolicy: Never # TODO: Remove this once docker images are deployed to Apache DockerHub post-graduation
@@ -1,8 +1,17 @@
apiVersion: extensions/v1beta1
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: gaas-ingress
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /
spec:
backend:
serviceName: gaas-svc
servicePort: 6956
rules:
- http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: gaas-svc
port:
number: 6956
@@ -30,9 +30,8 @@ spec:
name: gaas-config
containers:
- name: gobblin-service
image: will97/gobblin-as-a-service:latest
command: ["./bin/entrypoint.sh"]
args: ["--jvmopts", "-DmysqlCredentials.user=$(MYSQL_USERNAME) -DmysqlCredentials.password=$(MYSQL_PASSWORD)"]
image: apache/gobblin:latest
args: ["--mode", "gobblin-as-service", "--jvmopts", "-DmysqlCredentials.user=$(MYSQL_USERNAME) -DmysqlCredentials.password=$(MYSQL_PASSWORD) -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap"]
env:
- name: MYSQL_USERNAME
valueFrom:
@@ -46,9 +45,9 @@ spec:
key: password
volumeMounts:
- name: shared-jobs
mountPath: /tmp/gobblin-as-service/jobs
mountPath: /etc/gobblin-as-service/jobs
- name: flowconfig-templates
mountPath: /tmp/templateCatalog
mountPath: /etc/templateCatalog
- name: gaas-config
mountPath: /home/gobblin/conf/gobblin-as-service/application.conf
subPath: gaas-application.conf
@@ -86,10 +85,14 @@ spec:
name: standalone-config
containers:
- name: gobblin-standalone
image: will97/gobblin-standalone:latest
image: apache/gobblin:latest
args: ["--mode", "standalone"]
volumeMounts:
- name: shared-jobs
mountPath: /tmp/gobblin-standalone/jobs
mountPath: /etc/gobblin-standalone/jobs
- name: standalone-config
mountPath: /home/gobblin/conf/standalone/application.conf
subPath: standalone-application.conf
env:
- name: GOBBLIN_JOB_CONFIG_DIR
value: /etc/gobblin-standalone/jobs
@@ -18,7 +18,7 @@
# Sample configuration properties for the Gobblin Service

# Topology Catalog and Store
gobblin.service.work.dir=/tmp/gobblin-as-service
gobblin.service.work.dir=/etc/gobblin-as-service

# TopologySpec Factory
topologySpec.store.dir=${gobblin.service.work.dir}/topologySpecStore
@@ -30,7 +30,7 @@ spec:
persistentVolumeClaim:
claimName: mysql-pv-claim
containers:
- image: mysql:5.6.45
- image: mysql:8.0
name: mysql
env:
- name: MYSQL_RANDOM_ROOT_PASSWORD

0 comments on commit f4c7513

Please sign in to comment.