Parse minimal yaml instead of raw bash script

.gitignore

@@ -2,5 +2,5 @@ build/*
 .venv3
 .install
 .pre-commit
-development/nginx/command
+development/nginx/data/*
 coverage.out

README.md

@@ -55,18 +55,42 @@ Overview of what is needed:
 2. Start MQTT broker, data host for serving the script and minio storage
     * You can take advantage of `make development` command to create neccessary containers, inspect `development/podman-compose.yml` for more details
 3. Publish new message to broker
-    * Change `CLIENT_ID` on L16 in `development/python/mqtt_publish.py` file if needed
+    * [Optional] Change values in `development/python/mqtt_publish.py`
+      * `CLIENT_ID` - can be found in logs after running rhcd
+      * `SERVED_FILENAME` - one of the files inside `development/nginx/data`
     * Call `make publish`
 4. You should see logs in rhcd and file with stdout of your script uploaded to the minio storage
     * Go to http://localhost:9990/login and use credentials from `.env` file
 
 ### Bash script example
 
-*NOTE: This is subject to changes, right now worker is executing raw bash script, but in near future we expect that worker will execute bash script wrapped in signed yaml file.*
-
-Create a bash script file called `command` inside the folder `development/nginx` and place
-your code inside of it, like for example:
-
-```bash
-/usr/bin/convert2rhel --help
+Create or update a yaml file inside the folder `development/nginx/data/*`.
+Correct structure with exampe bash script can be seen below:
+
+```yml
+vars:
+  _insights_signature: |
+    ascii_armored gpg signature
+  _insights_signature_exclude: "/vars/insights_signature,/vars/content_vars"
+  content: |
+    #!/bin/sh
+    /usr/bin/convert2rhel --help
+  content_vars:
+    # variables that will be handed to the script as environment vars
+    # will be prefixed with RHC_WORKER_*
+    FOO: bar
+    BAR: foo
 ```
+### Environment variables
+
+Environment variables used by our worker are always prefixed with `RHC_WORKER_`.
+
+Use below variables to adjust worker behavior.
+* Related to logging
+  * `RHC_WORKER_LOG_FOLDER` - default is `"/var/log/rhc-worker-bash"`
+  * `RHC_WORKER_LOG_FILENAME` - default is `"rhc-worker-bash.log"`
+* Related to verification of yaml file containing bash script
+  * `RHC_WORKER_GPG_CHECK` - default is `"1"`
+  * `RHC_WORKER_VERIFY_YAML` - default is `"1"`
+* Related to script temporary location and execution
+  * `RHC_WORKER_TMP_DIR` - default is `"/var/lib/rhc-worker-bash"`

development/nginx/data/yaml-file

@@ -0,0 +1,18 @@
+vars:
+  # Signature to validate that no one tampered with script
+  _insights_signature: |
+    ascii_armored gpg signature
+  _insights_signature_exclude: "/vars/insights_signature,/vars/content_vars"
+  content: |
+    #!/bin/sh
+    data='{"alert": false, "summary": "convert2rhel did not detect issues", "report": "", "report_json": {"foo": "bar"}}'
+    /usr/bin/convert2rhel --help
+    /usr/bin/convert2rhel --version
+    echo "BEGIN MARKER"
+    echo "$data"
+    echo "END MARKER"
+  content_vars:
+    # variables that will be handed to the script as environment vars
+    # will be prefixed with RHC_WORKER_*
+    FOO: bar
+    BAR: foo

development/nginx/worker.conf

@@ -6,7 +6,7 @@ server {
     access_log  /var/log/nginx/host.access.log  main;
     error_log          /var/log/nginx/error.log debug;
 
-    location /command {
+    location /data/ {
         root       /www/data;
     }
 

development/podman-compose.yml

@@ -85,7 +85,7 @@ services:
   nginx:
     image: docker.io/nginx:latest
     volumes:
-      - ./nginx/command:/www/data/command:z
+      - ./nginx/data:/www/data/data:z
       - ./nginx/worker.conf:/etc/nginx/conf.d/default.conf:z
     ports:
       - "8000:80"

development/python/mqtt_publish.py

@@ -13,18 +13,21 @@ def get_ip_address():
   return host_ip
 
 # This is changed everytime you refresh the box and register the machine again.
-CLIENT_ID = "e9c0d844-1e0d-4829-9faf-5430aec16205"
+CLIENT_ID = "973afbce-19b4-4862-9d7a-6e9d8c410674"
 BROKER = '127.0.0.1'
 BROKER_PORT = 1883
 TOPIC = f"yggdrasil/{CLIENT_ID}/data/in"
 
+# NOTE: currently can be whatever you placed inside devleopment/nginx/data folder
+SERVED_FILENAME = "yaml-file"
+
 MESSAGE = {
   "type": "data",
   "message_id": str(uuid.uuid4()),
   "version": 1,
   "sent": "2021-01-12T14:58:13+00:00", # str(datetime.datetime.now().isoformat()),
   "directive": 'rhc-worker-bash',
-  "content": f'http://{get_ip_address()}:8000/command',
+  "content": f'http://{get_ip_address()}:8000/data/{SERVED_FILENAME}',
   "metadata": {
       "correlation_id": "00000000-0000-0000-0000-000000000000",
       "return_url": f'http://{get_ip_address()}:8000/api/ingress/v1/upload',

go.sum

@@ -139,6 +139,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

src/logger.go

@@ -32,7 +32,8 @@ func setupLogger(logFolder string, fileName string) *os.File {
 	if ok {
 		level, ok := log.ParseLevel(yggdLogLevel)
 		if ok != nil {
-			log.Errorf("Could not parse log level '%v'", yggdLogLevel)
+			log.Errorf("Could not parse log level '%v', setting the level to info", yggdLogLevel)
+			log.SetLevel(log.LevelInfo)
 		} else {
 			log.SetLevel(level)
 		}

src/logger_test.go

@@ -12,40 +12,45 @@ func TestSetupLogger(t *testing.T) {
 	// Create a temporary directory for the log folder
 	logFolderName := "log-test"
 	logFileName := "log-file"
-
 	defer os.RemoveAll(logFolderName)
 
-	// Mock the YGG_LOG_LEVEL environment variable
+	// Test case 1: YGG_LOG_LEVEL doesn't exist, info level should be set to info
+	setupLogger(logFolderName, logFileName)
+	level := log.CurrentLevel()
+	if log.CurrentLevel() != log.LevelInfo {
+		t.Errorf("Incorrect log level. Expected: %v, Got: %v", log.LevelInfo, level)
+	}
+	// Test case 2: Unparsable level in env variable
+	os.Setenv("YGG_LOG_LEVEL", "....")
+	setupLogger(logFolderName, logFileName)
+	level = log.CurrentLevel()
+	if log.CurrentLevel() != log.LevelInfo {
+		t.Errorf("Incorrect log level. Expected: %v, Got: %v", log.LevelInfo, level)
+	}
+
+	// Test case 3: Everything set up correctly
 	os.Setenv("YGG_LOG_LEVEL", "debug")
-	defer os.Unsetenv("YGG_LOG_LEVEL")
 
-	// Call the function being tested
 	logfile := setupLogger(logFolderName, logFileName)
-
-	// Verify that the log folder and file were created
 	if _, err := os.Stat(logFolderName); os.IsNotExist(err) {
 		t.Errorf("Log folder not created: %v", err)
 	}
-
 	logFilePath := filepath.Join(logFolderName, logFileName)
 	if _, err := os.Stat(logFilePath); os.IsNotExist(err) {
 		t.Errorf("Log file not created: %v", err)
 	}
-
-	// Verify that the log level was set correctly
-	level := log.CurrentLevel()
+	level = log.CurrentLevel()
 	if level != log.LevelDebug {
 		t.Errorf("Incorrect log level. Expected: %v, Got: %v", log.LevelDebug, level)
 	}
-
-	// Verify that the log flags were set correctly
 	flags := log.Flags()
 	expectedFlags := log.Lshortfile | log.LstdFlags
 	if flags != expectedFlags {
 		t.Errorf("Incorrect log flags. Expected: %v, Got: %v", expectedFlags, flags)
 	}
 
 	// Cleanup - close the log file
+	defer os.Unsetenv("YGG_LOG_LEVEL")
 	err := logfile.Close()
 	if err != nil {
 		t.Errorf("Failed to close log file: %v", err)

src/main.go

@@ -12,24 +12,23 @@ import (
 	"google.golang.org/grpc"
 )
 
+// Initialized in main
 var yggdDispatchSocketAddr string
 var logFolder string
 var logFileName string
 var temporaryWorkerDirectory string
+var shouldDoInsightsCoreGPGCheck string
+var shouldVerifyYaml string
 
 // main is the entry point of the application. It initializes values from the environment,
 // sets up the logger, establishes a connection with the dispatcher, registers as a handler,
 // listens for incoming messages, and starts accepting connections as a Worker service.
 // Note: The function blocks and runs indefinitely until the server is stopped.
 func main() {
-	// Get initialization values from the environment.
-	yggdDispatchSocketAddr, yggSocketAddrExists := os.LookupEnv("YGG_SOCKET_ADDR")
-	if !yggSocketAddrExists {
-		log.Fatal("Missing YGG_SOCKET_ADDR environment variable")
+	initializedOK, errorMsg := initializeEnvironment()
+	if errorMsg != "" && !initializedOK {
+		log.Fatal(errorMsg)
 	}
-	logFolder = getEnv("RHC_WORKER_BASH_LOG_FOLDER", "/var/log/rhc-worker-bash")
-	logFileName = getEnv("RHC_WORKER_BASH_LOG_FILENAME", "rhc-worker-bash.log")
-	temporaryWorkerDirectory = getEnv("RHC_WORKER_BASH_TMP_DIR", "/var/lib/rhc-worker-bash")
 
 	logFile := setupLogger(logFolder, logFileName)
 	defer logFile.Close()

src/runner.go

@@ -1,19 +1,134 @@
 package main
 
 import (
+	"fmt"
+	"os"
 	"os/exec"
+	"strings"
 
 	"git.sr.ht/~spc/go-log"
+	"gopkg.in/yaml.v3"
 )
 
-// Executes bash script and returns its standard output
-func executeScript(fileName string) string {
-	out, err := exec.Command("/bin/sh", fileName).Output()
+// Received Yaml data has to match the expected yamlConfig structure
+type yamlConfig struct {
+	Vars struct {
+		InsightsSignature        string            `yaml:"_insights_signature"`
+		InsightsSignatureExclude string            `yaml:"_insights_signature_exclude"`
+		Content                  string            `yaml:"content"`
+		ContentVars              map[string]string `yaml:"content_vars"`
+	} `yaml:"vars"`
+}
+
+// Verify that no one tampered with yaml file
+func verifyYamlFile(yamlData []byte) bool {
+
+	if shouldVerifyYaml != "1" {
+		log.Warnln("WARNING: Playbook verification disabled.")
+		return true
+	}
+
+	log.Infoln("Verifying yaml file...")
+	// --payload here will be a no-op because no upload is performed when using the verifier
+	//   but, it will allow us to update the egg!
+
+	args := []string{
+		"-m", "insights.client.apps.ansible.playbook_verifier",
+		"--quiet", "--payload", "noop", "--content-type", "noop",
+	}
+	env := os.Environ()
+
+	if shouldDoInsightsCoreGPGCheck == "0" {
+		args = append(args, "--no-gpg")
+		env = append(env, "BYPASS_GPG=True")
+	}
+
+	cmd := exec.Command("insights-client", args...)
+	cmd.Env = env
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		log.Errorln(err)
+		return false
+	}
+
+	// Send yaml data to the command's stdin
+	_, err = stdin.Write(yamlData)
+	if err != nil {
+		log.Errorln(err)
+		return false
+	}
+	stdin.Close()
+
+	output, err := cmd.Output()
+	if err != nil {
+		log.Errorln("ERROR: Unable to verify yaml file:", string(output), err)
+		return false
+	}
+	return true
+}
+
+// Parses given yaml data.
+// If signature is valid then extracts the bash script to temporary file,
+// sets env variables if present and then runs the script.
+// Return stdout of executed script or error message if the signature wasn't valid.
+func processSignedScript(yamlFileContet []byte) string {
+	signatureIsValid := verifyYamlFile(yamlFileContet)
+	if !signatureIsValid {
+		errorMsg := "Signature of yaml file is invalid"
+		log.Errorln(errorMsg)
+		return errorMsg
+	}
+	log.Infoln("Signature of yaml file is valid")
+
+	// Parse the YAML data into the yamlConfig struct
+	var yamlContent yamlConfig
+	err := yaml.Unmarshal(yamlFileContet, &yamlContent)
+	if err != nil {
+		log.Errorln(err)
+	}
+
+	// Set env variables
+	getEnvVarName := func(key string) string {
+		return fmt.Sprintf("RHC_WORKER_%s", strings.ToUpper(key))
+	}
+	for key, value := range yamlContent.Vars.ContentVars {
+		prefixedKey := getEnvVarName(key)
+		err := os.Setenv(prefixedKey, value)
+		if err != nil {
+			log.Errorln(err)
+		} else {
+			log.Infoln("Successfully set env variable", prefixedKey, "=", value)
+		}
+	}
+	defer func() {
+		for key := range yamlContent.Vars.ContentVars {
+			os.Unsetenv(getEnvVarName(key))
+		}
+	}()
+
+	// NOTE: just debug to see the values
+	log.Debugln("Insights Signature:", yamlContent.Vars.InsightsSignature)
+	log.Debugln("Insights Signature Exclude:", yamlContent.Vars.InsightsSignatureExclude)
+	log.Debugln("Script:", yamlContent.Vars.Content)
+	log.Debugln("Vars:")
+	for key, value := range yamlContent.Vars.ContentVars {
+		log.Debugln(" ", key, ":", value)
+	}
+
+	// Write the file contents to the temporary disk
+	log.Infoln("Writing temporary bash script")
+	scriptFileName := writeFileToTemporaryDir([]byte(yamlContent.Vars.Content), temporaryWorkerDirectory)
+	defer os.Remove(scriptFileName)
+
+	// Execute the script
+	log.Infoln("Executing bash script")
+
+	out, err := exec.Command("/bin/sh", scriptFileName).Output()
 	if err != nil {
 		log.Errorln("Failed to execute script: ", err)
 		return ""
 	}
 
-	log.Infoln("Bash script executed successfully.")
+	log.Infoln("Bash script executed successfully")
 	return string(out)
 }