Merge 30f483a into 97d9730

.github/workflows/actions.yml

@@ -0,0 +1,167 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+  pull_request:
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+
+
+  build_and_run:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Git Checkout
+        uses: actions/checkout@v3
+
+      - name: Building Python Environment
+        run: |
+          chmod 700 ./ci/env_setup.sh
+          ./ci/env_setup.sh
+      
+      - name: Running Docker Compose Orion-v2
+        run: |
+          cd ./ci/orion/
+          docker compose up -d
+          cd /home/runner/work/fiware-orion-pyspark-connector/fiware-orion-pyspark-connector
+          sleep 30
+          hostname=$(hostname -I | tr ' ' '\n' | grep 10)
+          sed -i "s/0.0.0.0/$hostname/g" ./ci/subscription.json
+          curl -vX POST http://localhost:1026/v2/entities/ -d @./ci/new_entity.json --header "Content-Type: application/json"
+          curl -vX POST http://localhost:1026/v2/subscriptions/ -d @./ci/subscription.json --header "Content-Type: application/json"
+
+
+      - name: V2 Tests
+        run: |
+          sleep 10
+          chmod 700 ./ci/v2_test/v2_test.sh
+          ./ci/v2_test/v2_test.sh
+          
+      - name: Run Test Newman
+        run: newman run ./ci/v2_test/Test.json --timeout-request 120000 --insecure --bail --verbose
+
+
+      - name: Raw Test
+        run: |
+          sleep 10
+          chmod 700 ./ci/raw_test/raw_test.sh
+          ./ci/raw_test/raw_test.sh
+
+
+      - name: Stopping v2 Container
+        run: |
+          cd ./ci/orion/
+          docker compose down 
+          cd /home/runner/work/fiware-orion-pyspark-connector/fiware-orion-pyspark-connector
+          
+          
+      - name: Running Docker Compose Orion-LD
+        run: |
+          cd ./ci/orion-ld/
+          docker compose up -d
+          cd /home/runner/work/fiware-orion-pyspark-connector/fiware-orion-pyspark-connector
+          sleep 30
+          hostname=$(hostname -I | tr ' ' '\n' | grep 10)
+          sed -i "s/0.0.0.0/$hostname/g" ./ci/ld_subscription.json
+          curl -vX POST http://localhost:1026/ngsi-ld/v1/entities/ -d @./ci/new_ld_entity.json --header "Content-Type: application/ld+json" --header "NGSILD-Tenant: orion"
+          curl -vX POST http://localhost:1026/ngsi-ld/v1/subscriptions/ -d @./ci/ld_subscription.json --header "Content-Type: application/ld+json" --header "NGSILD-Tenant: orion"
+      
+
+      - name: LD Test
+        run: |
+          sleep 10
+          chmod 700 ./ci/ld_test/ld_test.sh
+          ./ci/ld_test/ld_test.sh
+      
+
+      - name: Stopping LD Container
+        run: |
+          cd ./ci/orion-ld/
+          docker compose down
+          cd /home/runner/work/fiware-orion-pyspark-connector/fiware-orion-pyspark-connector
+  
+
+
+
+
+
+
+
+
+
+  Coverall-Testing:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Git Checkout
+        uses: actions/checkout@v3
+
+
+      - name: Building Testing Environment
+        run: |
+          chmod 700 ./Test/env_setup.sh
+          ./Test/env_setup.sh
+          
+      - name: Installing Coveralls
+        run: |
+          pip3 install coveralls
+          pip3 install pytest
+          
+      - name: Running Docker Compose Orion-v2
+        run: |
+          cd ./Test
+          docker compose up -d
+          cd /home/runner/work/fiware-orion-pyspark-connector/fiware-orion-pyspark-connector
+          sleep 20
+          hostname=$(hostname -I | tr ' ' '\n' | grep 10)
+          sed -i "s/0.0.0.0/$hostname/g" ./Test/subscription.json
+          curl -vX POST http://localhost:1026/v2/entities/ -d @./Test/new_entity.json --header "Content-Type: application/json"
+          curl -vX POST http://localhost:1026/v2/subscriptions/ -d @./Test/subscription.json --header "Content-Type: application/json"
+          
+      - name: Preparing Package
+        run: |
+          sleep 10
+          chmod 700 ./Test/prepare.sh
+          ./Test/prepare.sh
+          chmod 700 ./Test/RapidPUT.sh
+          ./Test/RapidPUT.sh &
+          
+      - name: Testing
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run:  |
+          cd ./Test/package
+          coverage run --source=./ start.py &
+          variable=$!
+          sleep 100
+          #kill $variable
+          coveralls --service=github
+          
+          
+      #- name: Push to Coveralls
+      # uses: coverallsapp/github-action@master
+      # with:
+      #    github-token: ${{ secrets.github_token }}
+
+
+      - name: Returning to home
+        run: cd /home/runner/work/fiware-orion-pyspark-connector/fiware-orion-pyspark-connector
+
+
+
+
+
+# Runs a set of commands using the runners shell
+#- name: Run a multi-line script
+#  run: |
+#    echo Add other actions to build,
+#    echo test, and deploy your project.

Test/RapidPUT.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+sleep 25
+curl -vX PUT http://localhost:1026/v2/entities/urn:ngsi-ld:Product:010/attrs/price/value -H "Content-Type: text/plain" -d "10"
+sleep 5
+curl -vX PUT http://localhost:1026/v2/entities/urn:ngsi-ld:Product:010/attrs/price/value -H "Content-Type: text/plain" -d "10"
+sleep 5
+curl -vX PUT http://localhost:1026/v2/entities/urn:ngsi-ld:Product:010/attrs/price/value -H "Content-Type: text/plain" -d "10"
+sleep 5
+curl -vX PUT http://localhost:1026/v2/entities/urn:ngsi-ld:Product:010/attrs/price/value -H "Content-Type: text/plain" -d "10"
+sleep 5
+curl -vX PUT http://localhost:1026/v2/entities/urn:ngsi-ld:Product:010/attrs/price/value -H "Content-Type: text/plain" -d "10"

Test/Test.json

@@ -0,0 +1,48 @@
+{
+	"info": {
+		"_postman_id": "ce492b8f-744f-4710-8bca-4647bacdb81c",
+		"name": "TestCollection",
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
+		"_exporter_id": "20344132"
+	},
+	"item": [
+		{
+			"name": "TestCollection",
+			"event": [
+				{
+					"listen": "test",
+					"script": {
+						"exec": [
+							"pm.test(\"Response check\", function(){",
+							"",
+							"    pm.response.to.have.status(200);",
+							"    pm.response.to.have.body('20')",
+							"})"
+						],
+						"type": "text/javascript"
+					}
+				}
+			],
+			"request": {
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:1026/v2/entities/urn:ngsi-ld:Product:010/attrs/price/value",
+					"protocol": "http",
+					"host": [
+						"localhost:1026"
+					],
+					"path": [
+						"v2",
+						"entities",
+						"urn:ngsi-ld:Product:010",
+						"attrs",
+						"price",
+						"value"
+					]
+				}
+			},
+			"response": []
+		}
+	]
+}

Test/Test.txt

@@ -0,0 +1 @@
+{ "value" : %%TOREPLACE%% }

Test/__init__.py

@@ -0,0 +1 @@
+

Test/connectorconf.py

@@ -0,0 +1,7 @@
+HTTPADDRESS = "0.0.0.0"
+HTTPPORT = 8061
+SOCKETADDRESS = 'localhost'
+SOCKETPORT = 9998
+REQUEST_COMPLETENESS = True
+SOCKET_BUFFER = 2048
+MAX_CONCURRENT_CONNECTIONS = 20

Test/docker-compose.yml

@@ -0,0 +1,35 @@
+version: "3"
+
+services:
+
+  orion:
+    hostname: orion
+    image: fiware/orion:latest
+    networks:
+      - hostnet
+      - ocbnet
+    ports:
+      - "1026:1026"
+    depends_on:
+      - orion_mongo
+    command: -statCounters -dbhost orion_mongo -logLevel INFO
+
+  orion_mongo:
+    hostname: orion_mongo
+    image: mongo:4.2
+    networks:
+      - ocbnet
+    ports:
+      - "27017:27017"
+    volumes:
+      - orion_mongo_data:/data/db
+      - orion_mongo_conf:/data/configdb
+    command: --nojournal
+
+volumes:
+  orion_mongo_data:
+  orion_mongo_conf:
+
+networks:
+  hostnet:
+  ocbnet:

Test/env_setup.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+echo "installing python 3.8"
+sudo apt update -y
+sudo apt-get install python3.8 -y
+sudo apt-get install python3-pip -y
+echo "installed python 3.8"
+
+echo "installing Java 11"
+sudo apt-get install -y java-11-openjdk-devel -y
+sudo apt-get install -y wget
+sudo apt-get install -y unzip
+echo "installed Java 11"
+
+
+echo "Installing Newman CLI..."
+sudo npm set strict-ssl false
+sudo npm install -g newman@5.2.2
+newman --version
+echo "Newman installed, READY TO TEST..."
+
+echo "installing Spark 3.2.1"
+wget https://dlcdn.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz
+tar xvf spark-3.2.1-bin-hadoop3.2.tgz
+mv spark-3.2.1-bin-hadoop3.2/ /opt/spark 
+export SPARK_HOME=/opt/spark
+export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin
+rm spark-3.2.1-bin-hadoop3.2.tgz
+echo "installed Spark 3.2.1"
+
+echo "installing python requirements: requests"
+pip3 install requests
+echo "installing python requirements: py4j"
+pip3 install 'py4j==0.10.9.3'
+echo "installing python requirements: pyspark"
+pip3 install 'pyspark==3.2.1'
+echo "installing python requirements: psutil"
+pip3 install psutil
+echo "installed every python dependency"
+echo "installing the connector"
+echo "downloading repository"
+wget https://github.com/Engineering-Research-and-Development/fiware-orion-pyspark-connector/archive/refs/heads/main.zip
+unzip main.zip
+echo "repository downloaded and unzipped"
+mkdir ci/PySpark/
+mv fiware-orion-pyspark-connector-main/receiver/connector_lib.py ./ci/PySpark/
+mv fiware-orion-pyspark-connector-main/receiver/connectorconf.py ./ci/PySpark/
+mv fiware-orion-pyspark-connector-main/replier/replier_lib.py ./ci/PySpark/
+mv fiware-orion-pyspark-connector-main/replier/replyconf.py ./ci/PySpark/
+rm -r fiware-orion-pyspark-connector-main
+rm main.zip
+echo "connector succesfully imported"
+

Test/new_entity.json

@@ -0,0 +1,6 @@
+{
+      "id":"urn:ngsi-ld:Product:010", "type":"Product",
+      "name":{"type":"Text", "value":"Lemonade"},
+      "size":{"type":"Text", "value": "S"},
+      "price":{"type":"Integer", "value": 99}
+}

Test/prepare.sh

@@ -0,0 +1,20 @@
+
+hostname=$(hostname -I | tr ' ' '\n' | grep 10)
+
+cd ./Test
+mkdir package
+
+sed -i "s/0.0.0.0/$hostname/g" connectorconf.py
+mv connectorconf.py ./package
+mv start.py ./package
+mv Test.txt ./package
+mv __init__.py ./package
+
+
+cd /home/runner/work/fiware-orion-pyspark-connector/fiware-orion-pyspark-connector/
+mv ./ci/PySpark/connector_lib.py ./Test/package
+mv ./ci/PySpark/replier_lib.py ./Test/package
+mv ./ci/PySpark/replyconf.py ./Test/package
+
+ls -l ./Test/package/
+

Test/start.py

@@ -0,0 +1,28 @@
+import connector_lib as connector
+import replier_lib as replier
+from pyspark import SparkContext, SparkConf, StorageLevel
+
+import os
+import psutil
+import time
+
+
+conf = SparkConf().setAppName("FIWARE Orion-PySpark Connector Demo").set("spark.hadoop.yarn.resourcemanager.address", "local[2]")
+sc = SparkContext(conf=conf)
+
+
+event, ssc = connector.Prime(sc, 5 , StorageLevel.MEMORY_AND_DISK_2)
+
+event = event.flatMap(lambda x: x.entities).map(lambda x: x.attrs['price'].value)
+#response = event.map(lambda x : replier.UnstructuredReplyToBroker('{ "value" :' + str(20) +' }'))
+#response2 = event.map(lambda x: replier.SemistructuredReplyToBroker("20", '{"value" : %%TOREPLACE%% }'))
+response3 = event.map(lambda x : replier.ReplyToBroker("20"))
+event.pprint()
+#response.pprint()
+#response2.pprint()
+response3.pprint()
+
+
+
+ssc.start()
+ssc.awaitTerminationOrTimeout(90)