Update the development environment

.gitignore

@@ -2,4 +2,4 @@ build/*
 .venv3
 .install
 .pre-commit
-python/command
+development/nginx/command

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 minimum_pre_commit_version: "2.9.0"
-exclude: ^(mosquitto/mosquitto.conf)
+exclude: ^(development/mosquitto/mosquitto.conf)
 
 repos:
   - repo: "https://github.com/golangci/golangci-lint"

Makefile

@@ -1,11 +1,11 @@
 .PHONY: \
-	install
-	clean
-	build
-	distribution-tarball
-	test
-	publish
-	broker
+	install \
+	clean \
+	build \
+	distribution-tarball \
+	test \
+	publish \
+	development
 
 # Project constants
 VERSION ?= 0.1
@@ -16,14 +16,6 @@ PIP ?= pip3
 VENV ?= .venv3
 PRE_COMMIT ?= pre-commit
 
-# Let the user specify PODMAN at the CLI, otherwise try to autodetect a working podman
-ifndef PODMAN
-	PODMAN := $(shell podman run --rm alpine echo podman 2> /dev/null)
-	ifndef PODMAN
-		DUMMY := $(warning podman is not detected. Majority of commands will not work. Please install and verify that podman --version works.)
-	endif
-endif
-
 all: clean build
 
 install: .install .pre-commit
@@ -65,5 +57,6 @@ test:
 publish:
 	. $(VENV)/bin/activate; python python/mqtt_publish.py
 
-broker:
-	@$(PODMAN) run -d -it -p 1883:1883 -p 9001:9001 -v $(PWD)/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf:Z eclipse-mosquitto
+development:
+	@podman-compose -f development/podman-compose.yml down
+	podman-compose -f development/podman-compose.yml up

README.md

@@ -1,4 +1,4 @@
-# RHC Bash Worker
+# RHC Worker Bash
 
 Remote Host Configuration (rhc) worker for executing bash scripts on hosts
 managed by Red Hat Insights.

development/.env

@@ -0,0 +1,5 @@
+MINIO_DATA_DIR=./minio-data
+MINIO_CONFIG_DIR=./minio-conf
+MINIO_ACCESS_KEY=oamg
+MINIO_SECRET_KEY=oamg
+INGRESS_VALID_UPLOAD_TYPES=advisor,qpc,tasks

development/nginx/command

@@ -0,0 +1 @@
+/usr/bin/convert2rhel --help

development/nginx/worker.conf

@@ -0,0 +1,30 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  localhost;
+
+    access_log  /var/log/nginx/host.access.log  main;
+    error_log          /var/log/nginx/error.log debug;
+
+    location /command {
+        root       /www/data;
+    }
+
+    location /api/ingress/v1/upload {
+        proxy_pass http://ingress:3000/api/ingress/v1/upload;
+
+        # May not need or want to set Host. Should default to the above hostname.
+        proxy_set_header          Host            $host;
+        proxy_set_header          X-Real-IP       $remote_addr;
+        proxy_set_header          X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    error_page  404              /404.html;
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}

development/podman-compose.yml

@@ -0,0 +1,107 @@
+# Taken from: https://github.com/RedHatInsights/insights-ingress-go/blob/master/development/local-dev-start.yml
+
+# This podman compose file stands up local dependencies for
+# Kafka, Zookeeper, and Minio and optionally insights-ingress-go itself.
+# Please consult README.md for bucket creation steps
+version: "3"
+services:
+  zookeeper:
+    image: docker.io/confluentinc/cp-zookeeper
+    environment:
+      - ZOOKEEPER_CLIENT_PORT=32181
+      - ZOOKEEPER_SERVER_ID=1
+    networks:
+      - worker
+
+  kafka:
+    image: confluentinc/cp-kafka
+    ports:
+      - 29092:29092
+    depends_on:
+      - zookeeper
+    environment:
+      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:29092
+      - KAFKA_BROKER_ID=1
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:32181
+      - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
+    networks:
+      - worker
+
+  minio:
+    image: docker.io/minio/minio
+    command: server --address 0.0.0.0:9000 --console-address 0.0.0.0:9990 /data
+    volumes:
+      - "./minio-conf:/mnt/config:z"
+      - "./minio-data/.minio:/mnt/data:z"
+    ports:
+      - 9000:9000
+      - 9990:9990
+    environment:
+      - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}
+      - MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
+      - MINIO_ROOT_USER=${MINIO_ACCESS_KEY}
+      - MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY}
+    networks:
+      - worker
+
+  minio-createbucket:
+    image: docker.io/minio/mc
+    depends_on:
+      - minio
+    restart: on-failure
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc config host add myminio http://minio:9000 ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} || exit 1;
+      /usr/bin/mc mb --ignore-existing myminio/insights-upload-perma;
+      /usr/bin/mc policy set public myminio/insights-upload-perma;
+      "
+    networks:
+      - worker
+
+  ingress:
+    image: quay.io/cloudservices/insights-ingress:latest
+    ports:
+      - 3000:3000
+    mem_limit: 512M
+    environment:
+      - INGRESS_STAGEBUCKET=insights-upload-perma
+        # INGRESS_VALID_UPLOAD_TYPES is required. This is derived from content type. ex: vnd.redhat.advisor.thing+tgz
+      - INGRESS_VALID_UPLOAD_TYPES=$INGRESS_VALID_UPLOAD_TYPES
+      - OPENSHIFT_BUILD_COMMIT=somestring
+      - INGRESS_METRICSPORT=8081
+      - INGRESS_MAXSIZE=104857600
+      - INGRESS_MINIODEV=true
+      - INGRESS_MINIOACCESSKEY=$MINIO_ACCESS_KEY
+      - INGRESS_MINIOSECRETKEY=$MINIO_SECRET_KEY
+      - INGRESS_MINIOENDPOINT=minio:9000
+      - INGRESS_AUTH=false
+    depends_on:
+      - kafka
+      - minio
+    networks:
+      - worker
+
+  nginx:
+    image: docker.io/nginx:latest
+    volumes:
+      - ./nginx/command:/www/data/command:z
+      - ./nginx/worker.conf:/etc/nginx/conf.d/default.conf:z
+    ports:
+      - "8000:80"
+    depends_on:
+      - ingress
+    networks:
+      - worker
+
+  mqtt:
+    image: eclipse-mosquitto
+    ports:
+      - 1883:1883
+      - 9001:9001
+    volumes: "./mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf:Z"
+    networks:
+      - worker
+
+networks:
+  worker: {}

python/mqtt_publish.py

@@ -1,25 +1,8 @@
 #!/usr/bin/env python3
 import json
-import os
 import socket
-import sys
-import time
 import uuid
-from http import server
 import paho.mqtt.client as mqtt
-import multiprocessing
-
-class CustomHandler(server.SimpleHTTPRequestHandler):
-  def __init__(self, request, client_address, server, *, directory = None):
-     super().__init__(request, client_address, server, directory=directory)
-
-  def handle_one_request(self):
-      super().handle_one_request()
-      sys.exit(0)
-
-def start_server(host, port):
-  httpd = server.HTTPServer((host, port), CustomHandler)
-  httpd.serve_forever()
 
 def get_ip_address():
   host_ip = ""
@@ -30,46 +13,32 @@ def get_ip_address():
   return host_ip
 
 # This is changed everytime you refresh the box and register the machine again.
-CLIENT_ID = "22e7b41c-944c-4778-81cf-facb6444d1a0"
+CLIENT_ID = "e73f3b91-0766-4d33-8d4c-8a529f5f1bc7"
 BROKER = '127.0.0.1'
 BROKER_PORT = 1883
 TOPIC = f"yggdrasil/{CLIENT_ID}/data/in"
 
 MESSAGE = {
   "type": "data",
   "message_id": str(uuid.uuid4()),
-  # client_uuid doesn't seemt to be us  ed
-  # "client_uuid": CLIENT_ID,
   "version": 1,
   "sent": "2021-01-12T14:58:13+00:00", # str(datetime.datetime.now().isoformat()),
   "directive": 'rhc-worker-bash',
-  "content": f'http://{get_ip_address()}:8000/python/command',
+  "content": f'http://{get_ip_address()}:8000/command',
   "metadata": {
-    "report_file": "/var/log/convert2rhel/convert2rhel-report.json",
-    "return_url": 'http://raw.example.com/return'
+      "correlation_id": "00000000-0000-0000-0000-000000000000",
+      "return_url": f'http://{get_ip_address()}:8000/api/ingress/v1/upload',
+      "return_content_type": "application/vnd.redhat.tasks.filename+tgz"
   }
 }
 
 
 def main():
-  if not os.path.exists("python/command"):
-    print("You must create a python/command file in order to continue.")
-    sys.exit(1)
-
-  process = multiprocessing.Process(target=start_server, args=('0.0.0.0', 8000))
-  process.start()
-
-  print("Sleeping for 1 second to wait for the server")
-  time.sleep(1)
-
   client = mqtt.Client()
   client.connect(BROKER, BROKER_PORT, 60)
   client.publish(TOPIC, json.dumps(MESSAGE), 1, False)
   print("Published message to MQTT, serving content.")
 
-  process.join()
-
-
 
 if __name__ == "__main__":
    main()

src/util.go

@@ -48,7 +48,7 @@ func readOutputFile(filePath string) (*bytes.Buffer, string) {
 	writer := multipart.NewWriter(body)
 
 	h := make(textproto.MIMEHeader)
-	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, "file", "convert2rhel-report.json.tar.gz"))
+	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, "file", "rhc-worker-bash-output.tar.gz"))
 	h.Set("Content-Type", "application/vnd.redhat.tasks.filename+tgz")
 	part, err := writer.CreatePart(h)
 	if err != nil {