Merge branch 'develop' into misc/ansible-remove

.gitignore

@@ -21,9 +21,9 @@ reports/
 *.log
 logs/
 kilda-bins/
-tar/
 *.retry
 services/src/api/target/
+.venv
 
 # autogenerated config files
 /services/logstash/etc/logstash/conf.d/logstash.conf

.travis.yml

@@ -3,11 +3,17 @@ services:
 
 jobs:
   include:
-    - stage: ansible-test
-      language: python
+    - stage: confd-test
+      before_install:
+        - wget https://github.com/kelseyhightower/confd/releases/download/v0.16.0/confd-0.16.0-linux-amd64
+        - sudo mkdir -p /opt/confd/bin
+        - sudo mv confd-0.16.0-linux-amd64 /opt/confd/bin/confd
+        - sudo chmod +x /opt/confd/bin/confd
+        - export PATH="$PATH:/opt/confd/bin"
+
       script:
-        - pip install ansible
         - make update-props-dryrun
+        - make update-props
 
     - stage: tests
       name: "Sonar"
@@ -22,12 +28,17 @@ jobs:
           token:
             secure: "ekznTpLDEd2x5SMcxPE609zYDv+dDMbgyQOWGjYdRsMrjeAqJvlRPz3NFqqLK0+RXzGYO+TOfZf1rUpv5B47n4j1U8O5wDoXw5ZXAXM8hHNbz4qqMZdfXAOyTGMsqrBHLC0omFxWNfTc9QMQtdWbqgSU3KzGDg/0J3pypQoCAP52e5bSzSBchVXR5xmInaFYlGHZfsZpKuSavRki3hvwlM1N6F3abQQgQq3XVH05KHerhv/3AQ9gjHv4O0K1oRt0CJo1TztMIlRB70MY4JMv8Bpf09PzteP/fjZBlB7vIKZ1TQOph9mMYJ8guXatTvZ7Ym2Lkihq2/KvG6p650oeQxTE0b4j6aqbZPN+C3o9INMnA80chZzBySGjYleNOBPXl5bDouW94HKfvBdQFfFRFuHS280fVrYvO11IYdkglcxnk3YhMcJO/iJuH7IiUeWLQ6vpVOPJzDBt7QQqhTFWySidEnmQdnlotQigXpgZ9h98qyHR3fXVFAArWX1bcSA9WAIZHAFJwjVSimVCq5SJDPhgkcyyzauInNiFb1fPMjDscYfCXpK7ws9ydzMTVV+kQ+2x8JO9jJwDOzUkGHV83zheO3TzSCfuANBhP1GwkcSVvgEcrc5A5eosbC3DDcQpK7KRX7QHKsnSetqovh8MoFeNG4nYObOC4oyPZT6AXf4="
 
+      before_cache:
+        - sudo chown -R travis:travis $HOME/.m2/repository
+        - sudo chown -R travis:travis $HOME/.sonar/cache
+
       script:
         # JaCoCo is used to have code coverage, the agent has to be activated
         - export MAVEN_OPTS="$MAVEN_OPTS -DinteractiveMode=false -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
         - make -C services/src
-        - cd services/src
-        - mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar
+        - mvn -f services/src/pom.xml clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar
+        - make -C services/wfm
+        - mvn -f services/wfm/pom.xml clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar
 
       cache:
         directories:
@@ -40,8 +51,11 @@ jobs:
       jdk: oraclejdk8
 
       before_install:
-        - sudo pip install --upgrade pip
-        - sudo pip install ansible
+        - wget https://github.com/kelseyhightower/confd/releases/download/v0.16.0/confd-0.16.0-linux-amd64
+        - sudo mkdir -p /opt/confd/bin
+        - sudo mv confd-0.16.0-linux-amd64 /opt/confd/bin/confd
+        - sudo chmod +x /opt/confd/bin/confd
+        - export PATH="$PATH:/opt/confd/bin"
 
       script:
         - export MAVEN_OPTS="$MAVEN_OPTS -DinteractiveMode=false -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"

Makefile

@@ -2,7 +2,7 @@
 default: build-latest run-dev
 
 build-base:
-	base/hacks/shorm.requirements.download.sh
+	base/hacks/storm.requirements.download.sh
 	docker build -t kilda/base-ubuntu:latest base/kilda-base-ubuntu/
 	docker build -t kilda/zookeeper:latest services/zookeeper
 	docker build -t kilda/kafka:latest services/kafka
@@ -31,7 +31,7 @@ up-test-mode:
 up-log-mode: up-test-mode
 	docker-compose logs -f
 
-# keeping run-test for backwards compatibility (documentation) .. should depracate
+# keeping run-test for backwards compatibility (documentation) .. should deprecate
 run-test: up-log-mode
 
 clean-sources:
@@ -50,7 +50,6 @@ update-msg:
 
 update: update-parent update-msg update-pce
 
-
 compile:
 	$(MAKE) -C services/src
 	$(MAKE) -C services/wfm all-in-one
@@ -72,7 +71,9 @@ avoid-port-conflicts:
 clean-test:
 	docker-compose down
 	docker-compose rm -fv
-	docker volume list -q | grep kilda | xargs docker volume rm
+	docker volume list -q | grep kilda | xargs -r docker volume  rm
+
+clean: clean-sources clean-test
 
 update-props:
 	confd -onetime -confdir ./confd/ -backend file -file ./confd/vars/main.yaml -sync-only
@@ -119,3 +120,4 @@ sec: update
 .PHONY: up-test-mode up-log-mode run-test clean-test
 .PHONY: atdd smoke perf sec
 .PHONY: clean-sources unit update
+.PHONY: clean

README.md

@@ -24,15 +24,23 @@ Also, don't forget to install ansible. This tool is used for creating config/pro
 pip install ansible
 ```
 
-### How to Build Kilda Controller
+### How to build Kilda Controller
 
 From the base directory run the following command:
 
 ```
 make build-latest
 ```
 
-### How to run Kilda
+### How to clean Kilda Controller
+
+From the base directory run the following command:
+
+```
+make clean
+```
+
+### How to run Kilda Controller
 
 __NB: To run Kilda, you should have built it already (ie the previous section).__
 This is particularly important because docker-compose will expect some of the

confd/vars/main.yaml

@@ -34,8 +34,8 @@ kilda_mininet_rest_port: "38080"
 kilda_topology_engine_endpoint: "http://127.0.0.1"
 kilda_topology_engine_rest_endpoint: "http://127.0.0.1"
 kilda_topology_engine_rest_port: "80"
-kilda_northbound_endpoint: "http://127.0.0.1"
-kilda_northbound_rest_port: "8088"
+kilda_northbound_endpoint: "http://northbound.pendev"
+kilda_northbound_rest_port: "8080"
 
 kilda_northbound_username: "kilda"
 kilda_northbound_password: "kilda"

docker-compose.yml

@@ -154,7 +154,6 @@ services:
     container_name: wfm
     build:
       context: services/wfm
-    image: "kilda/wfm:${full_build_number:-latest}"
     depends_on:
       kafka:
         condition: service_healthy
@@ -319,7 +318,7 @@ services:
     build:
       context: services/src/northbound/
     ports:
-      - "8088:8080"
+      - "8080:8080"
     environment:
       REST_USERNAME: 'kilda'
       REST_PASSWORD: 'kilda'

docs/README.md

@@ -1,9 +1,9 @@
 
 # How to contribute to OpenKilda project
-The folder contains OpenKilda contributor licenses agreemetns and tips for contributors.
+The folder contains OpenKilda contributor licenses agreements and tips for contributors.
 
 # OpenKilda design documents
-The folder contains the main use case digrams and their descriptions.
+The folder contains the main use case diagrams and their descriptions.
 
 # Release notes
-The folder contains OpenKilda release notes
+The folder contains OpenKilda release notes.

docs/design/README.MD

@@ -1,6 +1,6 @@
 
 # Introduction
-This document describes OpenKilda design including main use cases, components design and their interactions.
+This document describes OpenKilda design including main use cases, components design, and their interactions.
 
 Overall components [interaction](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/northbound-and-flow-topology.svg) among Northbound, Kafka, Flow topology, Cache topology and Topology engine.
 
@@ -9,9 +9,9 @@ Overall components [interaction](https://github.com/telstra/open-kilda/blob/deve
 The document describes main use cases for the flow operations create, delete, read, update and ping:
 - [Create Flow](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/flow-crud-create-full.png) sequence diagram 
 - [Create Flow](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/flow-crud-create-full.txt) sequence source file
-- [Delete Flow](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/flow-crud-delete-full.png) seqiecne diagram
+- [Delete Flow](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/flow-crud-delete-full.png) sequence diagram
 - [Delete Flow](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/flow-crud-delete-full.txt) sequence source file
-2. [Flow Validation](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/flow-ping-manual.md) mechanism description designed for sending pings along flow path in order to verify its latency and switch availability along the flow path
+2. [Flow Validation](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/flow-ping-manual.md) mechanism description designed for sending pings along a flow path in order to verify its latency and switch availability along the flow path
 
 ## Switch operations
 1. [Delete meter](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/meter-delete.md) document describes the implementation of deleting a meter from the switch.
@@ -22,8 +22,8 @@ network discovery design implemented in OpenKilda.
 
 ## Flow Topology
 [The document](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/flow-topology-high-level.md) 
-contains high level flow topology design implemented in Storm.
-There is also overall [description](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/storm-topology-management.md)
+contains high-level flow topology design implemented in Storm.
+There is also an overall [description](https://github.com/telstra/open-kilda/blob/develop/docs/design/usecase/storm-topology-management.md)
 of the patterns as to how topologies work in OpenKilda in Storm.
 
 ## Kafka

docs/design/flow-ping/flow-ping.md

@@ -0,0 +1,95 @@
+# Flow pings
+## The idea
+Use our link validation mechanism to validate the whole flow. We can inject
+validation packet on edge switch, it will be "routed" using flow rules on all 
+intermediate switches and cached on the edge switch on other side of flow
+("packet out" message must specify special out port called "TABLE" to pass
+packet via existing on switch open flow rules).
+
+## Goals
+* measure flow latency
+* one more way to verify switch availability (detect control plane issues)
+
+# Ping topology
+## Periodic pings
+Periodic pings are initiated by kilda system periodically, with interval defined by `flow.ping.interval` configuration option. Pings for both directions for all existing flows are created on each iteration.
+
+If ping success - it's latency is stored into OTSDB, in metric `pen.flow.latency`, with tags flowid and direction.
+
+If there is no successful pings for `flow.ping.fail.delay` seconds - flow failure will be reported in logs also `PingReport` message will be send into kafka topic defined in `kafka.topic.flow.status` configuration option.
+
+To simplify flow status log filtration, all flow status log message are prefixed with `{FLOW-PING}` keyword.
+
+Example of `PingReport` message:
+```json
+{
+  "clazz": "org.openkilda.messaging.info.InfoMessage",
+  "payload": {
+    "clazz": "org.openkilda.messaging.info.flow.FlowPingReport",
+    "report": {
+      "flowid": "flowping-alpha",
+      "status": "OPERATIONAL"
+    },
+    "timestamp": 1531157435563
+  },
+  "timestamp": 1531157435563,
+  "correlation_id": "435ed47d-2438-4037-b432-d1d0cb01a709"
+}
+```
+
+![pediodic pings](./periodic-ping-sequence-diagram.png "Periodic ping sequence diagram")
+
+## On demand pings
+This kind of flow ping is initiated via NorthBound API. The only mandatory input parameter if flowId. On this request kilda will initiate 2 pings, one in forward direction and one in reverse direction. 
+
+NorthBound endpoint - ulr path `/flows/{flow_id}/ping`, method PUT. In request body you can pass additional ping options.
+
+Request body example:
+```json
+{
+  "timeout": 2000
+}
+```
+
+`timeout` - how long time wait for response (in milliseconds). If not defined - the default value 2000 will be used.
+
+Successful ping call response:
+```json
+{
+  "flow_id": "flowping-alpha",
+  "forward": {
+    "ping_success": true,
+    "error": null,
+    "latency": 5
+  },
+  "reverse": {
+    "ping_success": true,
+    "error": null,
+    "latency": 2
+  },
+  "error": null
+}
+```
+
+Failure in forward direction:
+```json
+{
+  "flow_id": "flowping-alpha",
+  "forward": {
+    "ping_success": false,
+    "error": "No ping for reasonable time",
+    "latency": 0
+  },
+  "reverse": {
+    "ping_success": true,
+    "error": null,
+    "latency": 1
+  },
+  "error": null
+}
+```
+
+![on-demand pings](./on-demand-ping-sequence-diagram.png "On demand ping sequence diagram")
+
+## Common part for all kinds of pings
+![common pings](./ping-sequence-diagram.png "All ping kinds common part sequence diagram")

docs/design/flow-ping/on-demand-ping-sequence-diagram.txt

@@ -0,0 +1,72 @@
+@startuml
+title ON DEMAND flow pings
+
+participant NB
+
+control kilda.ping
+
+participant InputDecoder
+participant InputRouter
+
+participant FlowFetcher
+participant Neo4j
+
+participant PING
+participant ResultDispatcher
+participant OnDemandResultManager
+participant GroupCollector
+
+participant NorthboundEncoder
+
+control kilda.northbound
+
+NB -> kilda.ping: json{FlowPingRequest}
+kilda.ping -> InputDecoder: json{FlowPingRequest}
+
+activate InputDecoder
+InputDecoder -> InputRouter: FlowPingRequest
+deactivate InputDecoder
+
+activate InputRouter
+InputRouter -> FlowFetcher: FlowPingRequest
+deactivate InputRouter
+
+activate FlowFetcher
+FlowFetcher -> Neo4j: fetch flow\nby flowId
+activate Neo4j
+Neo4j -> FlowFetcher: BidirectionalFlow
+deactivate Neo4j
+
+note right of FlowFetcher: make PingContext(kind=ON_DEMAND)
+FlowFetcher -> PING: PingContext
+deactivate FlowFetcher
+
+activate PING
+note right of PING: see details\nin "The Ping" diagram
+PING -> ResultDispatcher: PingContext
+deactivate PING
+
+activate ResultDispatcher
+ResultDispatcher -> OnDemandResultManager: PingContext
+deactivate ResultDispatcher
+
+activate OnDemandResultManager
+OnDemandResultManager -> GroupCollector: PingContext (groupId grouping)
+deactivate OnDemandResultManager
+
+activate GroupCollector
+alt when ping group is completed
+    GroupCollector -> OnDemandResultManager: Group
+end
+deactivate GroupCollector
+
+activate OnDemandResultManager
+note right of OnDemandResultManager: make FlowPingResponse
+OnDemandResultManager -> NorthboundEncoder: FlowPingResponse
+deactivate OnDemandResultManager
+
+activate NorthboundEncoder
+NorthboundEncoder -> kilda.northbound: json{FlowPingResponse}
+deactivate NorthboundEncoder
+
+@enduml