Merge remote-tracking branch 'origin/feature/erasure' into feature/er…

…asure
eclipse-ditto · Oct 19, 2018 · 84384c8 · 84384c8
2 parents 8c4898e + 8da1ed2
commit 84384c8
Show file tree

Hide file tree

Showing 175 changed files with 5,233 additions and 909 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -15,12 +15,12 @@ script:
   - if [ -n "$TRAVIS_TAG" ]; then
     export IMAGE_TAG=$TRAVIS_TAG;
     docker login -u "$DOCKERHUB_USER" -p "$DOCKERHUB_PASSWORD";
-    docker build -f services/Dockerfile --build-arg SERVICE_STARTER=ditto-services-policies-starter     --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-policies:$IMAGE_TAG      -t eclipse/ditto-policies:latest .;
-    docker build -f services/Dockerfile --build-arg SERVICE_STARTER=ditto-services-things-starter       --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-things:$IMAGE_TAG        -t eclipse/ditto-things:latest .;
-    docker build -f services/Dockerfile --build-arg SERVICE_STARTER=ditto-services-thingsearch-starter  --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-things-search:$IMAGE_TAG -t eclipse/ditto-things-search:latest .;
-    docker build -f services/Dockerfile --build-arg SERVICE_STARTER=ditto-services-concierge-starter    --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-concierge:$IMAGE_TAG     -t eclipse/ditto-concierge:latest .;
-    docker build -f services/Dockerfile --build-arg SERVICE_STARTER=ditto-services-gateway-starter      --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-gateway:$IMAGE_TAG       -t eclipse/ditto-gateway:latest .;
-    docker build -f services/Dockerfile --build-arg SERVICE_STARTER=ditto-services-connectivity-starter --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-connectivity:$IMAGE_TAG  -t eclipse/ditto-connectivity:latest .;
+    docker build -f services/src/Dockerfile --build-arg SERVICE_STARTER=ditto-services-policies-starter     --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-policies:$IMAGE_TAG      -t eclipse/ditto-policies:latest .;
+    docker build -f services/src/Dockerfile --build-arg SERVICE_STARTER=ditto-services-things-starter       --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-things:$IMAGE_TAG        -t eclipse/ditto-things:latest .;
+    docker build -f services/src/Dockerfile --build-arg SERVICE_STARTER=ditto-services-thingsearch-starter  --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-things-search:$IMAGE_TAG -t eclipse/ditto-things-search:latest .;
+    docker build -f services/src/Dockerfile --build-arg SERVICE_STARTER=ditto-services-concierge-starter    --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-concierge:$IMAGE_TAG     -t eclipse/ditto-concierge:latest .;
+    docker build -f services/src/Dockerfile --build-arg SERVICE_STARTER=ditto-services-gateway-starter      --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-gateway:$IMAGE_TAG       -t eclipse/ditto-gateway:latest .;
+    docker build -f services/src/Dockerfile --build-arg SERVICE_STARTER=ditto-services-connectivity-starter --build-arg SERVICE_VERSION=$IMAGE_TAG   -t eclipse/ditto-connectivity:$IMAGE_TAG  -t eclipse/ditto-connectivity:latest .;
     docker push eclipse/ditto-policies:$IMAGE_TAG;
     docker push eclipse/ditto-policies:latest;
     docker push eclipse/ditto-things:$IMAGE_TAG;

diff --git a/bom/pom.xml b/bom/pom.xml
@@ -38,8 +38,8 @@
         <!-- ### Compile dependencies versions -->
         <minimal-json.version>0.9.5</minimal-json.version>
         <typesafe-config.version>1.3.2</typesafe-config.version>
-        <akka.version>2.5.13</akka.version>
-        <akka-http.version>10.1.3</akka-http.version>
+        <akka.version>2.5.17</akka.version>
+        <akka-http.version>10.1.5</akka-http.version>
         <akka-persistence-mongo.version>2.0.9</akka-persistence-mongo.version>
         <akka-management.version>0.14.0</akka-management.version>
         <alpakka-mqtt.version>0.20</alpakka-mqtt.version>
@@ -140,6 +140,19 @@
                 <artifactId>akka-cluster-sharding_${scala.version}</artifactId>
                 <version>${akka.version}</version>
             </dependency>
+            <dependency>
+                <groupId>com.typesafe.akka</groupId>
+                <artifactId>akka-remote_${scala.version}</artifactId>
+                <version>${akka.version}</version>
+                <exclusions>
+                    <!-- exclude netty from akka-remote - we use aeron remoting instead -->
+                    <!-- due to license issues with the old netty version we are not allowed to use it -->
+                    <exclusion>
+                        <groupId>io.netty</groupId>
+                        <artifactId>netty</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
             <dependency>
                 <groupId>com.typesafe.akka</groupId>
                 <artifactId>akka-slf4j_${scala.version}</artifactId>
@@ -637,6 +650,11 @@
                 <artifactId>ditto-services-utils-config</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.eclipse.ditto</groupId>
+                <artifactId>ditto-services-utils-ddata</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.eclipse.ditto</groupId>
                 <artifactId>ditto-services-utils-devops</artifactId>
@@ -652,6 +670,11 @@
                 <artifactId>ditto-services-utils-jwt</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.eclipse.ditto</groupId>
+                <artifactId>ditto-services-utils-namespaces</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.eclipse.ditto</groupId>
                 <artifactId>ditto-services-utils-persistence</artifactId>

diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml
@@ -25,6 +25,7 @@ services:
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError
+      - REMOTING_IDLE_CPU_LEVEL=1
 
   things:
     init: true
@@ -41,6 +42,7 @@ services:
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError
+      - REMOTING_IDLE_CPU_LEVEL=1
 
   things-search:
     init: true
@@ -57,6 +59,7 @@ services:
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError
+      - REMOTING_IDLE_CPU_LEVEL=1
 
   concierge:
     init: true
@@ -73,6 +76,7 @@ services:
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError
+      - REMOTING_IDLE_CPU_LEVEL=1
 
   connectivity:
     init: true
@@ -90,6 +94,7 @@ services:
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError
+      - REMOTING_IDLE_CPU_LEVEL=1
 
   gateway:
     init: true
@@ -108,14 +113,15 @@ services:
       - TZ=Europe/Berlin
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
+      - REMOTING_IDLE_CPU_LEVEL=1
       - ENABLE_DUMMY_AUTH=true
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError
 
   swagger-ui:
     image: docker.io/swaggerapi/swagger-ui:v2.2.9
     volumes:
-       - ../documentation/src/main/resources/openapi:/usr/share/nginx/html/openapi:ro
-       - ../documentation/src/main/resources/images:/usr/share/nginx/html/images:ro
+       - ../../documentation/src/main/resources/openapi:/usr/share/nginx/html/openapi:ro
+       - ../../documentation/src/main/resources/images:/usr/share/nginx/html/images:ro
        - ./swagger-index.html:/usr/share/nginx/html/index.html:ro
     environment:
       - API_URL=/apidoc/openapi/ditto-api-1.yml
@@ -127,7 +133,7 @@ services:
        - ./nginx.htpasswd:/etc/nginx/nginx.htpasswd:ro
        - ./nginx-cors.conf:/etc/nginx/nginx-cors.conf:ro
        - ./index.html:/etc/nginx/html/index.html:ro
-       - ../documentation/src/main/resources/images:/etc/nginx/html/images:ro
+       - ../../documentation/src/main/resources/images:/etc/nginx/html/images:ro
     ports:
       - "8080:80"
     depends_on:

diff --git a/deployment/docker/sandbox/docker-compose.yml b/deployment/docker/sandbox/docker-compose.yml
@@ -25,6 +25,7 @@ services:
       - TZ=Europe/Berlin
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
+      - REMOTING_IDLE_CPU_LEVEL=1
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError -Xtune:virtualized -Dakka.coordinated-shutdown.exit-jvm=on -Dakka.cluster.shutdown-after-unsuccessful-join-seed-nodes=120s
       - MONGO_COLLECTION_NAME_SUFFIX_CLASS=
 
@@ -42,6 +43,7 @@ services:
       - TZ=Europe/Berlin
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
+      - REMOTING_IDLE_CPU_LEVEL=1
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError -Xtune:virtualized -Dakka.coordinated-shutdown.exit-jvm=on -Dakka.cluster.shutdown-after-unsuccessful-join-seed-nodes=120s
       - MONGO_COLLECTION_NAME_SUFFIX_CLASS=
 
@@ -59,6 +61,7 @@ services:
       - TZ=Europe/Berlin
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
+      - REMOTING_IDLE_CPU_LEVEL=1
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError -Xtune:virtualized -Dakka.coordinated-shutdown.exit-jvm=on -Dakka.cluster.shutdown-after-unsuccessful-join-seed-nodes=120s
 
   concierge:
@@ -75,6 +78,7 @@ services:
       - TZ=Europe/Berlin
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
+      - REMOTING_IDLE_CPU_LEVEL=1
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError -Xtune:virtualized -Dakka.coordinated-shutdown.exit-jvm=on -Dakka.cluster.shutdown-after-unsuccessful-join-seed-nodes=120s
 
   connectivity:
@@ -92,6 +96,7 @@ services:
       - TZ=Europe/Berlin
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
+      - REMOTING_IDLE_CPU_LEVEL=1
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError -Xtune:virtualized -Dakka.coordinated-shutdown.exit-jvm=on -Dakka.cluster.shutdown-after-unsuccessful-join-seed-nodes=120s
 
   gateway:
@@ -109,15 +114,16 @@ services:
       - TZ=Europe/Berlin
       - INSTANCE_INDEX=1
       - BIND_HOSTNAME=0.0.0.0
+      - REMOTING_IDLE_CPU_LEVEL=1
       - ENABLE_DUMMY_AUTH=true
       - DEVOPS_SECURE_STATUS=false
       - IBM_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError -Xtune:virtualized -Dakka.coordinated-shutdown.exit-jvm=on -Dakka.cluster.shutdown-after-unsuccessful-join-seed-nodes=120s
 
   swagger-ui:
     image: docker.io/swaggerapi/swagger-ui:v2.2.9
     volumes:
-       - ../../documentation/src/main/resources/openapi:/usr/share/nginx/html/openapi:ro
-       - ../../documentation/src/main/resources/images:/usr/share/nginx/html/images:ro
+       - ../../../documentation/src/main/resources/openapi:/usr/share/nginx/html/openapi:ro
+       - ../../../documentation/src/main/resources/images:/usr/share/nginx/html/images:ro
        - ../swagger-index.html:/usr/share/nginx/html/index.html:ro
     environment:
        - API_URL=/apidoc/openapi/ditto-api-1.yml
@@ -138,7 +144,7 @@ services:
        - ./nginx-devops.htpasswd:/etc/nginx/nginx-devops.htpasswd:ro
        - ./html:/etc/nginx/html
        - ../nginx-cors.conf:/etc/nginx/nginx-cors.conf:ro
-       - ../../documentation/src/main/resources/images:/etc/nginx/html/images:ro
+       - ../../../documentation/src/main/resources/images:/etc/nginx/html/images:ro
        - /opt/letsencrypt/src/letsencrypt/letsencrypt-site/.well-known:/etc/nginx/html/.well-known:ro
        - /opt/letsencrypt/src/production/dh-param/dhparam-2048.pem:/etc/ssl/certs/dhparam-2048.pem
        - /opt/docker-volumes/etc/letsencrypt/live/ditto.eclipse.org/fullchain.pem:/etc/letsencrypt/live/ditto.eclipse.org/fullchain.pem

diff --git a/deployment/kubernetes/README.md b/deployment/kubernetes/README.md
@@ -37,40 +37,40 @@ minikube start
 This is necessary for the pods to access the Kubernetes API and then build the akka cluster.
 ```bash
 cd <DITTO_PATH>
-kubectl apply -f kubernetes/pod-reader-role.yaml
+kubectl apply -f deployment/kubernetes/pod-reader-role.yaml
 ```
 
 ### Create configuration mappings
 ```bash
-kubectl create configmap nginx-conf --from-file=kubernetes/nginx/nginx.conf
-kubectl create configmap nginx-cors --from-file=kubernetes/nginx/nginx-cors.conf
-kubectl create configmap nginx-htpasswd --from-file=kubernetes/nginx/nginx.htpasswd
-kubectl create configmap nginx-index --from-file=kubernetes/nginx/index.html
+kubectl create configmap nginx-conf --from-file=deployment/kubernetes/nginx/nginx.conf
+kubectl create configmap nginx-cors --from-file=deployment/kubernetes/nginx/nginx-cors.conf
+kubectl create configmap nginx-htpasswd --from-file=deployment/kubernetes/nginx/nginx.htpasswd
+kubectl create configmap nginx-index --from-file=deployment/kubernetes/nginx/index.html
 kubectl create configmap swagger-ui-api --from-file=$PWD/documentation/src/main/resources/openapi
 ```
 
 ### Start Eclipse Ditto
 
 #### Start MongoDB
 ```bash
-kubectl apply -f kubernetes/mongodb/mongodb.yaml
+kubectl apply -f deployment/kubernetes/mongodb/mongodb.yaml
 ```
 
 #### Start Ditto services
 ```bash
-kubectl apply -f kubernetes/ditto/ditto-cluster.yaml
-# Start ditto services with an alternative version e.g. 0.1.0-SNAPSHOT
-# cat kubernetes/ditto/ditto-cluster.yaml | sed s/latest/0.1.0-SNAPSHOT/ | kubectl apply -f -
+kubectl apply -f deployment/kubernetes/ditto/ditto-cluster.yaml
+# Start ditto services with an alternative version e.g. 0-SNAPSHOT
+# cat kubernetes/ditto/ditto-cluster.yaml | sed s/latest/0-SNAPSHOT/ | kubectl apply -f -
 ```
 
 #### Start Swagger UI
 ```bash
-kubectl apply -f kubernetes/swagger/swagger.yaml
+kubectl apply -f deployment/kubernetes/swagger/swagger.yaml
 ```
 
 #### Start Reverse Proxy (nginx)
 ```bash
-kubectl apply -f kubernetes/nginx/nginx.yaml
+kubectl apply -f deployment/kubernetes/nginx/nginx.yaml
 ```
 
 ### Use Eclipse Ditto
@@ -83,4 +83,4 @@ minikube service ditto
 minikube dashboard
 ```
 
-Have Fun!
+Have Fun!
diff --git a/documentation/src/main/resources/_posts/2018-10-16-example-mqtt-bidirectional.md b/documentation/src/main/resources/_posts/2018-10-16-example-mqtt-bidirectional.md
@@ -0,0 +1,58 @@
+---
+title: "Example demonstrating connectivity to an MQTT broker"
+published: true
+permalink: 2018-10-16-example-mqtt-bidirectional.html
+layout: post
+author: thomas_jaeckle
+tags: [blog]
+hide_sidebar: true
+sidebar: false
+toc: false
+---
+
+Eclipse Ditto can now connect to MQTT 3.1.1 brokers.
+
+Perfect timing to happily welcome a new colleague to our team behind Eclipse Ditto: [David](https://github.com/joosdavid). 
+
+In order to get familiar with Ditto and the development with digital twins, David was assigned with one of his first 
+tasks to get his hands on one of our newest features, [MQTT connectivity](connectivity-protocol-bindings-mqtt.html) 
+which was released recently with milestone [0.8.0-M2](2018-09-27-milestone-announcement-080-M2.html).
+
+On his journey into digital twin land he made a great example with an ESP8266 powered board connected via an MQTT broker
+to Ditto and published it to the 
+[Eclipse Ditto examples repository](https://github.com/eclipse/ditto-examples/tree/master/mqtt-bidirectional):
+
+> This example is about how to communicate between device and solution in a two way pattern through Ditto using MQTT. 
+  This means we will add a policy, a thing and a MQTT connection to Ditto. 
+  When Ditto ist set up and working, we will create real world device ("octopus board") and connect it to it's 
+  digital twin via MQTT.
+  <br/><br/>
+  At the end, we will create a basic frontend webapp. 
+  The webapp will automatically connect to Ditto when you type in your credentials, automatically pull your things 
+  and show them in a list.
+  <br/>
+  You can create, modify and delete devices in the webapp and if there is a real world device connected to the thing 
+  in the list, you can send it a command message to control any feature on it. 
+  The list of things will always be up-to-date when you listen to server-sent-events, which you can activate easily 
+  with pressing a button.
+
+{% include external_image.html
+href="https://raw.githubusercontent.com/eclipse/ditto-examples/master/mqtt-bidirectional/img/diagram.jpg" 
+alt="Eclipse Ditto bidirectional MQTT diagram" 
+max-width=800 
+caption="Source: https://github.com/eclipse/ditto-examples" %}
+
+> We will use an Octopus-board with an ESP8266 on it. It has several sensors built in, but for simplicity we will
+ just use it's temperature and altitude sensor. 
+ <br/>
+ To show the functionality of Eclipse Ditto messages, we will switch on/off a LED on the Octopus-board through it.
+
+
+If you have any wishes, improvements, are missing something
+or just want to get in touch with us, you can use one of
+our [feedback channels](https://eclipse.org/ditto/feedback.html).
+
+
+{% include image.html file="ditto.svg" alt="Ditto" max-width=500 %}
+--<br/>
+The Eclipse Ditto team
diff --git a/documentation/src/main/resources/openapi/ditto-api-1.yml b/documentation/src/main/resources/openapi/ditto-api-1.yml
@@ -204,7 +204,7 @@ paths:
         If no ACL is provided, a default ACL with an entry for the authorized subject with all permissions set to `true` will be created.
 
 
-        Use the placeholder `${request.subjectId}` in order to let the backend insert the authenticated subjectId of the HTTP request.
+        Use the placeholder `{{ request:subjectId }}` in order to let the backend insert the authenticated subjectId of the HTTP request.
 
         ### Permissions for updating an existing Thing
 
@@ -426,7 +426,7 @@ paths:
               JSON representation of the Access Control List (ACL).
 
 
-              Use the placeholder `${request.subjectId}` in order to let the backend insert the authenticated subjectId of the HTTP request.
+              Use the placeholder `{{ request:subjectId }}` in order to let the backend insert the authenticated subjectId of the HTTP request.
             required: true
             schema:
               $ref: '#/definitions/Acl'

diff --git a/documentation/src/main/resources/openapi/ditto-api-2.yml b/documentation/src/main/resources/openapi/ditto-api-2.yml
@@ -1867,7 +1867,7 @@ paths:
             JSON representation of the Policy.
 
 
-            Use the placeholder `${request.subjectId}` in order to let the backend insert the authenticated subjectId of the HTTP request.
+            Use the placeholder `{{ request:subjectId }}` in order to let the backend insert the authenticated subjectId of the HTTP request.
           required: true
           schema:
             $ref: '#/definitions/Policy'
@@ -2026,7 +2026,7 @@ paths:
             JSON representation of the Policy entries.
 
 
-            Use the placeholder `${request.subjectId}` in order to let the backend insert the authenticated subjectId of the HTTP request.
+            Use the placeholder `{{ request:subjectId }}` in order to let the backend insert the authenticated subjectId of the HTTP request.
           required: true
           schema:
             $ref: '#/definitions/PolicyEntries'
@@ -2133,7 +2133,7 @@ paths:
             JSON representation of the Policy entry.
 
 
-            Use the placeholder `${request.subjectId}` in order to let the backend insert the authenticated subjectId of the HTTP request.
+            Use the placeholder `{{ request:subjectId }}` in order to let the backend insert the authenticated subjectId of the HTTP request.
           required: true
           schema:
             $ref: '#/definitions/PolicyEntry'
@@ -2294,7 +2294,7 @@ paths:
             JSON representation of the Subjects.
 
 
-            Use the placeholder `${request.subjectId}` in order to let the backend insert the authenticated subjectId of the HTTP request.
+            Use the placeholder `{{ request:subjectId }}` in order to let the backend insert the authenticated subjectId of the HTTP request.
           required: true
           schema:
             $ref: '#/definitions/Subjects'
@@ -3021,7 +3021,7 @@ definitions:
           The initial Policy to create for this Thing. This will create a separate Policy entity managed by resource `/policies/{thingId}`.
 
 
-          Use the placeholder `${request.subjectId}` in order to let the backend insert the authenticated subjectId of the HTTP request.
+          Use the placeholder `{{ request:subjectId }}` in order to let the backend insert the authenticated subjectId of the HTTP request.
       policyId:
         type: string
         description: >-