Skip to content
Permalink
Browse files
[FLINK-22185] Add k8s example
  • Loading branch information
igalshilman authored and tzulitai committed Apr 12, 2021
1 parent ebbf53e commit 42c69e87d79728da2b77706993d2a70e5fd28633
Showing 14 changed files with 705 additions and 0 deletions.
@@ -0,0 +1,6 @@
target/
*.iml
.idea/
venv/*
__pycahe__/*
.DS_Store
@@ -0,0 +1,6 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: statefun

@@ -0,0 +1,46 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: minio
namespace: statefun
spec:
replicas: 1
selector:
matchLabels:
app: statefun
component: minio
template:
metadata:
labels:
app: statefun
component: minio
namespace: statefun
spec:
containers:
- name: minio
image: minio/minio
command: ["minio"]
args: ["server", "/data"]
ports:
- containerPort: 9000
name: endpoint
livenessProbe:
tcpSocket:
port: 9000
initialDelaySeconds: 30
periodSeconds: 60
---
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: statefun
spec:
type: ClusterIP
ports:
- name: endpoint
port: 9000
selector:
app: statefun
component: minio

@@ -0,0 +1,109 @@
---
apiVersion: v1
kind: Service
metadata:
labels:
app: zookeeper-service
name: zookeeper-service
namespace: statefun
spec:
type: NodePort
ports:
- name: zookeeper-port
port: 2181
targetPort: 2181
selector:
app: zookeeper

---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: zookeeper
name: zookeeper
namespace: statefun
spec:
replicas: 1
selector:
matchLabels:
app: zookeeper
template:
metadata:
labels:
app: zookeeper
spec:
containers:
- image: library/zookeeper:3.4.13
imagePullPolicy: IfNotPresent
name: zookeeper
ports:
- containerPort: 2181
env:
- name: ZOO_MY_ID
value: "1"
---
apiVersion: v1
kind: Service
metadata:
name: kafka
namespace: statefun
labels:
app: kafka
spec:
ports:
- port: 9092
name: plaintext
- port: 9999
name: jmx
clusterIP: None
selector:
app: kafka
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kafka
namespace: statefun
spec:
selector:
matchLabels:
app: kafka
replicas: 1
template:
metadata:
labels:
app: kafka
spec:
containers:
- name: kafka
image: wurstmeister/kafka:2.11-2.0.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9092
name: plaintext
- containerPort: 9999
name: jmx
env:
- name: KAFKA_BROKER_ID
value: "1"
- name: KAFKA_ADVERTISED_HOST_NAME
value: "kafka.statefun.svc.cluster.local"
- name: KAFKA_ADVERTISED_PORT
value: "9092"
- name: KAFKA_ZOOKEEPER_CONNECT
value: "zookeeper-service:2181"
- name: KAFKA_LISTENERS
value: "PLAINTEXT://:9092"
- name: KAFKA_JMX_OPTS
value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=9999 -Dcom.sun.management.jmxremote.rmi.port=9999 -Djava.rmi.server.hostname=127.0.0.1"
- name: JMX_PORT
value: "9999"
- name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
value: "true"
- name: KAFKA_LOG_RETENTION_MS
value: "600000"
- name: KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS
value: "60000"
- name: KAFKA_TRANSACTION_MAX_TIMEOUT_MS
value: "3600000"
@@ -0,0 +1,30 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM python:3.9-slim-buster

RUN mkdir -p /app
WORKDIR /app

COPY requirements.txt /app
RUN pip install -r requirements.txt

COPY functions.py /app

EXPOSE 8000

CMD ["python3", "/app/functions.py"]

@@ -0,0 +1,22 @@
.PHONY: configmap image service

image:
export $(shell minikube docker-env)
docker build . -t functions

service:
kubectl create -n statefun -f functions-service.yaml

upgrade:
kubectl rollout restart deployment/functions

logs:
kubectl logs -f -l component=functions -n statefun

pods:
kubectl get pods -n statefun -l component=functions

delete:
kubectl delete deployment functions -n statefun
kubectl delete svc functions -n statefun

@@ -0,0 +1,44 @@
apiVersion: v1
kind: Service
metadata:
name: functions
namespace: statefun
spec:
type: ClusterIP
ports:
- name: endpoint
port: 8000
selector:
app: statefun
component: functions
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: functions
namespace: statefun
spec:
replicas: 1
selector:
matchLabels:
app: statefun
component: functions
template:
metadata:
labels:
app: statefun
component: functions
spec:
containers:
- name: functions
image: functions
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8000
name: endpoint
livenessProbe:
tcpSocket:
port: 8000
initialDelaySeconds: 30
periodSeconds: 60

@@ -0,0 +1,46 @@
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
import json

from statefun import *
import asyncio
from aiohttp import web

functions = StatefulFunctions()


@functions.bind(typename="example/hello")
async def hello(context: Context, message: Message):
arg = message.raw_value().decode('utf-8')
print(f"Hello from {context.address.id}: you wrote {arg}!", flush=True)



handler = RequestReplyHandler(functions)

async def handle(request):
req = await request.read()
res = await handler.handle_async(req)
return web.Response(body=res, content_type="application/octet-stream")


app = web.Application()
app.add_routes([web.post('/statefun', handle)])

if __name__ == '__main__':
web.run_app(app, port=8000)
@@ -0,0 +1,18 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

aiohttp
apache-flink-statefun
@@ -0,0 +1,35 @@
apiVersion: v1
kind: ConfigMap
metadata:
namespace: statefun
name: module-config
data:
module.yaml: |+
version: "3.0"
module:
meta:
type: remote
spec:
endpoints:
- endpoint:
meta:
kind: http
spec:
functions: example/*
urlPathTemplate: http://functions.statefun.svc.cluster.local:8000/statefun
timeouts:
call: 2min
ingresses:
- ingress:
meta:
type: io.statefun.kafka/ingress
id: example/in
spec:
address: kafka.statefun.svc.cluster.local:9092
consumerGroupId: my-group-id
topics:
- topic: names
valueType: example/MyMessage
targets:
- example/hello

0 comments on commit 42c69e8

Please sign in to comment.