From 245ff025d10bf993e421e02d8d2107ef9f81a6dd Mon Sep 17 00:00:00 2001 From: Mykola Date: Sat, 9 Mar 2019 14:47:57 -0500 Subject: [PATCH 1/7] Add security checks, extend CI to public repo --- ci/.gitignore | 1 + ci/README.md | 13 +++++--- ci/ci_busy.sh | 26 ++++++++-------- ci/clone-and-checkout-pr.py | 11 ++++--- ci/manager.sh | 18 ++++++----- ci/webhook-receiver.py | 59 +++++++++++++++++++++++++++++++++---- 6 files changed, 95 insertions(+), 33 deletions(-) diff --git a/ci/.gitignore b/ci/.gitignore index 594e0099f..65908f45e 100644 --- a/ci/.gitignore +++ b/ci/.gitignore @@ -1,4 +1,5 @@ config +webhook-config.json githubcreds mykeyfile repository diff --git a/ci/README.md b/ci/README.md index 9415feca0..47e5c0db2 100644 --- a/ci/README.md +++ b/ci/README.md @@ -3,12 +3,12 @@ ### Setting up CI Run a Flask server that listens for new events from github, will get triggered when a new PR is created or when keyword `@onvm` is mentioned. ```sh -python3 webhook-receiver.py 0.0.0.0 8080 @onvm +python3 webhook-receiver.py 0.0.0.0 8080 @onvm webhook-config.json ``` To run CI tests manually, requires a config file, the github PR ID, request message and a response message. ```sh -./manager.sh +./manager.sh ``` ### Usage @@ -34,7 +34,6 @@ The CI process can be broken into multiple steps: WORKER_LIST=("WORKER_1_IP WORKER_1_KEY", "WORKER_2_IP WORKER_2_KEY", ...) GITHUB_CREDS=path_to_creditential_file REPO_OWNER="OWNER_STRING" - REPO_NAME="NAME_STRING" ``` Config file example: @@ -42,7 +41,13 @@ The CI process can be broken into multiple steps: WORKER_LIST=("nimbnode42 nn42_key") GITHUB_CREDS=githubcreds REPO_OWNER="sdnfv" - REPO_NAME="openNetVM-dev" + ``` + + Webhook json config example + ``` + { + "secret": "look_at_me_i'm_a_secret_key" + } ``` GITHUB_CREDS file example: diff --git a/ci/ci_busy.sh b/ci/ci_busy.sh index 1dd1e732a..59d510f39 100755 --- a/ci/ci_busy.sh +++ b/ci/ci_busy.sh @@ -29,18 +29,26 @@ fi if [[ -z "$3" ]] then - echo "ERROR: Missing third argument, Request body!" + echo "ERROR: Missing third argument, Repo name!" exit 1 else - REQUEST=$3 + REPO_NAME=$3 fi if [[ -z "$4" ]] then - echo "ERROR: Missing fourth argument, POST_MSG!" + echo "ERROR: Missing fourth argument, Request body!" exit 1 else - POST_MSG=$4 + REQUEST=$4 +fi + +if [[ -z "$5" ]] +then + echo "ERROR: Missing fifth argument, POST_MSG!" + exit 1 +else + POST_MSG=$5 fi . $1 # source the variables from config file @@ -48,24 +56,18 @@ fi print_header "Checking Required Variables" - if [[ -z "$GITHUB_CREDS" ]] +if [[ -z "$GITHUB_CREDS" ]] then echo "ERROR: GITHUB_CREDS not provided" exit 1 fi - if [[ -z "$REPO_OWNER" ]] +if [[ -z "$REPO_OWNER" ]] then echo "ERROR: REPO_OWNER not provided" exit 1 fi - if [[ -z "$REPO_NAME" ]] -then - echo "ERROR: REPO_NAME not provided" - exit 1 -fi - print_header "Posting Message in Comments on GitHub" python3 post-msg.py $GITHUB_CREDS "{\"id\": $PR_ID,\"request\":\"$REQUEST\"}" $REPO_OWNER $REPO_NAME "$POST_MSG" check_exit_code "ERROR: Failed to post results to GitHub" diff --git a/ci/clone-and-checkout-pr.py b/ci/clone-and-checkout-pr.py index e55ea9a16..b61ff6cd9 100644 --- a/ci/clone-and-checkout-pr.py +++ b/ci/clone-and-checkout-pr.py @@ -41,10 +41,13 @@ cmd = "git clone " + str(repo.clone_url) + " repository" child = pexpect.spawn(cmd) -child.expect("Username.*") -child.sendline(username + "\n") -child.expect("Password.*") -child.sendline(password + "\n") + +if '-dev' in REPO_NAME: + child.expect("Username.*") + child.sendline(username + "\n") + child.expect("Password.*") + child.sendline(password + "\n") + child.interact() print(pexpect.run("git checkout " + branch_name, cwd="./repository")) diff --git a/ci/manager.sh b/ci/manager.sh index d09b2d612..01978c807 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -31,10 +31,18 @@ fi if [[ -z "$3" ]] then - echo "ERROR: Missing third argument, Request body!" + echo "ERROR: Missing third argument, Repo name!" exit 1 else - REQUEST=$3 + REPO_NAME=$3 +fi + +if [[ -z "$4" ]] +then + echo "ERROR: Missing fourth argument, Request body!" + exit 1 +else + REQUEST=$4 fi . $1 # source the variables from config file @@ -60,12 +68,6 @@ then exit 1 fi -if [[ -z "$REPO_NAME" ]] -then - echo "ERROR: REPO_NAME not provided" - exit 1 -fi - print_header "Cleaning up Old Results" sudo rm -f *.txt diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 9739918c8..abffd95e9 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -1,6 +1,9 @@ from flask import Flask, jsonify, request + import requests +from ipaddress import ip_address, ip_network +import hmac import json import sys import pprint @@ -12,6 +15,30 @@ app = Flask(__name__) +def verify_request_ip(request): + src_ip = ip_address(u'{}'.format(request.access_route[0])) + valid_ips = requests.get('https://api.github.com/meta').json()['hooks'] + + for ip in valid_ips: + if src_ip in ip_network(ip): + return True + + return False + +def verify_request_secret(request): + header_signature = request.headers.get('X-Hub-Signature') + if header_signature is None: + return False + + signature = header_signature.split('=')[1] + + mac = hmac.new(str(secret).encode('utf-8'), msg=request.data, digestmod='sha1') + if not hmac.compare_digest(mac.hexdigest(), signature): + return False + + return True + + # returns extracted data if it is an event for a PR creation or PR comment creation # if it is a PR comment, only return extracted data if it contains the required keyword specified by the global var # if it doesn't contain the keyword or is not the correct type of event, return None @@ -22,6 +49,7 @@ def filter_to_prs_and_pr_comments(json): if json['action'] == 'opened' and 'pull_request' in json and 'base' in json['pull_request']: branch_name = json['pull_request']['base']['label'] + repo_name = json['repository']['name'] if branch_name is None: return None @@ -32,12 +60,13 @@ def filter_to_prs_and_pr_comments(json): return { "id": number, + "repo": repo_name, "branch": branch_name, "body": "In response to PR creation" } if json['action'] == 'created' and 'issue' in json and json['issue']['state'] == 'open' and 'pull_request' in json['issue'] and 'comment' in json: - + repo_name = json['repository']['name'] comment_txt = json['comment']['body'] if KEYWORD not in comment_txt: return None @@ -51,6 +80,7 @@ def filter_to_prs_and_pr_comments(json): return { "id": number, + "repo": repo_name, "body": comment_txt } @@ -58,6 +88,18 @@ def filter_to_prs_and_pr_comments(json): @app.route(EVENT_URL, methods=['POST']) def init_ci_pipeline(): + + if not verify_request_ip(request): + print("Incoming webkooh not from a valid Github address") + return jsonify({ + "success": True + }) + + if not verify_request_secret(request): + print("Incoming webhook secret doesn't match configured secret") + return jsonify({ + "success": True + }) extracted_data = filter_to_prs_and_pr_comments(request.json) if extracted_data is not None: @@ -73,10 +115,10 @@ def init_ci_pipeline(): if (out): print("Can't run CI, another CI run in progress") - os.system("./ci_busy.sh config {} \"{}\" \"Another CI run in progress, please try again in 15 minutes\"" - .format(extracted_data['id'], extracted_data['body'])) + os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"Another CI run in progress, please try again in 15 minutes\"" + .format(extracted_data['id'], extracted_data['repo'], extracted_data['body'])) else: - os.system("./manager.sh config {} \"{}\"".format(extracted_data['id'], extracted_data['body'])) + os.system("./manager.sh config {} \"{}\" \"{}\"".format(extracted_data['id'], extracted_data['repo'], extracted_data['body'])) else: print("Data did not match filter, SKIP CI") @@ -93,7 +135,7 @@ def status(): if __name__ == "__main__": global KEYWORD - if(len(sys.argv) != 4): + if(len(sys.argv) != 5): print("Invalid arguments!") sys.exit(1) @@ -101,4 +143,11 @@ def status(): port = sys.argv[2] KEYWORD = sys.argv[3] + with open (sys.argv[4], 'r') as cfg: + webhook_config = json.load(cfg) + secret = webhook_config['secret'] + if secret is None: + print("No secret found in webhook config") + sys.exit(1) + app.run(host=host, port=port) From fc3dc19ac16cdb2aef7b04c6b174ac5889b944f3 Mon Sep 17 00:00:00 2001 From: Mykola Date: Thu, 14 Mar 2019 00:49:19 -0400 Subject: [PATCH 2/7] Added authorized users --- ci/webhook-receiver.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index abffd95e9..8fb3dd22d 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -50,6 +50,7 @@ def filter_to_prs_and_pr_comments(json): if json['action'] == 'opened' and 'pull_request' in json and 'base' in json['pull_request']: branch_name = json['pull_request']['base']['label'] repo_name = json['repository']['name'] + user_name = json['pull_request']['user']['login'] if branch_name is None: return None @@ -62,12 +63,15 @@ def filter_to_prs_and_pr_comments(json): "id": number, "repo": repo_name, "branch": branch_name, + "user": user_name, "body": "In response to PR creation" } if json['action'] == 'created' and 'issue' in json and json['issue']['state'] == 'open' and 'pull_request' in json['issue'] and 'comment' in json: repo_name = json['repository']['name'] comment_txt = json['comment']['body'] + user_name = json['comment']['user']['login'] + if KEYWORD not in comment_txt: return None if json['sender']['login'] == CI_NAME: @@ -81,6 +85,7 @@ def filter_to_prs_and_pr_comments(json): return { "id": number, "repo": repo_name, + "user": user_name, "body": comment_txt } @@ -103,6 +108,14 @@ def init_ci_pipeline(): extracted_data = filter_to_prs_and_pr_comments(request.json) if extracted_data is not None: + if (extracted_data['user'] in authorized_users): + print("This is an authorized user") + else: + print("This user is not authorized") + return jsonify({ + "success": True + }) + print("Data matches filter, we should RUN CI") print(extracted_data) @@ -146,6 +159,7 @@ def status(): with open (sys.argv[4], 'r') as cfg: webhook_config = json.load(cfg) secret = webhook_config['secret'] + authorized_users = webhook_config['authorized-users'] if secret is None: print("No secret found in webhook config") sys.exit(1) From bfa58732085cb522c0360f15c800c951712fcc52 Mon Sep 17 00:00:00 2001 From: Mykola Date: Thu, 14 Mar 2019 00:58:36 -0400 Subject: [PATCH 3/7] Authorized user log --- ci/webhook-receiver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 8fb3dd22d..5c9500f20 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -108,10 +108,12 @@ def init_ci_pipeline(): extracted_data = filter_to_prs_and_pr_comments(request.json) if extracted_data is not None: - if (extracted_data['user'] in authorized_users): + if (extracted_data['repo'] == 'openNetVM-dev' or extracted_data['user'] in authorized_users): print("This is an authorized user") else: - print("This user is not authorized") + print("ERROR: This user is not authorized") + os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"User not authorized to run CI, please contact one of the repo maintainers\"" + .format(extracted_data['id'], extracted_data['repo'], extracted_data['body'])) return jsonify({ "success": True }) From 0028b57be84f3e1528aa06330874698304788e5e Mon Sep 17 00:00:00 2001 From: Mykola Date: Wed, 27 Mar 2019 12:45:07 -0400 Subject: [PATCH 4/7] Add secret encryption --- ci/.gitignore | 3 +++ ci/webhook-receiver.py | 47 +++++++++++++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/ci/.gitignore b/ci/.gitignore index 65908f45e..7e89ed673 100644 --- a/ci/.gitignore +++ b/ci/.gitignore @@ -1,5 +1,8 @@ config webhook-config.json +encrypted_secret.bin +private.pem +public.pem githubcreds mykeyfile repository diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 5c9500f20..95a41dd03 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -3,6 +3,9 @@ import requests from ipaddress import ip_address, ip_network +from Crypto.PublicKey import RSA +from Crypto.Cipher import AES, PKCS1_OAEP + import hmac import json import sys @@ -15,6 +18,32 @@ app = Flask(__name__) +def decrypt_secret(): + secret_file = open(webhook_config['secret-file'], "rb") + private_key = RSA.import_key(open(webhook_config['private-key-file']).read()) + + enc_session_key, nonce, tag, ciphertext = \ + [ secret_file.read(x) for x in (private_key.size_in_bytes(), 16, 16, -1) ] + + # Decrypt the session key with the private RSA key + cipher_rsa = PKCS1_OAEP.new(private_key) + session_key = cipher_rsa.decrypt(enc_session_key) + + # Decrypt the data with the AES session key + cipher_aes = AES.new(session_key, AES.MODE_EAX, nonce) + data = cipher_aes.decrypt_and_verify(ciphertext, tag) + + # Clear memory + secret_file = private_key = enc_session_key = nonce = tag = ciphertext = None + del secret_file + del private_key + del enc_session_key + del nonce + del tag + del ciphertext + + return data + def verify_request_ip(request): src_ip = ip_address(u'{}'.format(request.access_route[0])) valid_ips = requests.get('https://api.github.com/meta').json()['hooks'] @@ -31,13 +60,18 @@ def verify_request_secret(request): return False signature = header_signature.split('=')[1] + secret = decrypt_secret() - mac = hmac.new(str(secret).encode('utf-8'), msg=request.data, digestmod='sha1') - if not hmac.compare_digest(mac.hexdigest(), signature): - return False + mac = hmac.new(secret, msg=request.data, digestmod='sha1') + secret_comparison = hmac.compare_digest(mac.hexdigest(), signature) - return True + # Memory cleanup + header_signature = secret = signature = None + del header_signature + del secret + del signature + return secret_comparison # returns extracted data if it is an event for a PR creation or PR comment creation # if it is a PR comment, only return extracted data if it contains the required keyword specified by the global var @@ -160,10 +194,9 @@ def status(): with open (sys.argv[4], 'r') as cfg: webhook_config = json.load(cfg) - secret = webhook_config['secret'] authorized_users = webhook_config['authorized-users'] - if secret is None: - print("No secret found in webhook config") + if authorized_users is None: + print("No authroized users found in webhook config") sys.exit(1) app.run(host=host, port=port) From d446af667558bc40d3a17353d4de420ce658d89c Mon Sep 17 00:00:00 2001 From: Mykola Date: Thu, 28 Mar 2019 01:15:17 -0400 Subject: [PATCH 5/7] Add logging --- ci/.gitignore | 1 + ci/webhook-receiver.py | 117 +++++++++++++++++++++++------------------ 2 files changed, 67 insertions(+), 51 deletions(-) diff --git a/ci/.gitignore b/ci/.gitignore index 7e89ed673..100ae87b0 100644 --- a/ci/.gitignore +++ b/ci/.gitignore @@ -1,3 +1,4 @@ +access_log config webhook-config.json encrypted_secret.bin diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 95a41dd03..10b035fb6 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -12,12 +12,27 @@ import pprint import os import subprocess +import logging EVENT_URL = "/github-webhook" CI_NAME="onvm" app = Flask(__name__) +logging.getLogger('werkzeug').setLevel(logging.ERROR) +logging.basicConfig(filename="access_log", filemode='a', + format='%(asctime)s, %(name)s %(levelname)s %(message)s', + datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO) + +def get_request_info(request_ctx): + return "Request details: IP: {}, User: {}, Repo: {}, ID: {}, Body: {}.".format(request_ctx['src_ip'], request_ctx['user'], request_ctx['repo'], request_ctx['id'], request_ctx['body']) + +def log_access_granted(request_ctx, custom_msg): + logging.info("Access GRANTED: {}. {}".format(custom_msg, get_request_info(request_ctx))) + +def log_access_denied(request_ctx, custom_msg): + logging.info("Access DENIED: {}. {}".format(custom_msg, get_request_info(request_ctx))) + def decrypt_secret(): secret_file = open(webhook_config['secret-file'], "rb") private_key = RSA.import_key(open(webhook_config['private-key-file']).read()) @@ -44,8 +59,9 @@ def decrypt_secret(): return data -def verify_request_ip(request): - src_ip = ip_address(u'{}'.format(request.access_route[0])) +def verify_request_ip(request_ctx): + src_ip = request_ctx['src_ip'] + print (src_ip) valid_ips = requests.get('https://api.github.com/meta').json()['hooks'] for ip in valid_ips: @@ -54,15 +70,15 @@ def verify_request_ip(request): return False -def verify_request_secret(request): - header_signature = request.headers.get('X-Hub-Signature') +def verify_request_secret(request_ctx): + header_signature = request_ctx['X-Hub-Signature'] if header_signature is None: return False signature = header_signature.split('=')[1] secret = decrypt_secret() - mac = hmac.new(secret, msg=request.data, digestmod='sha1') + mac = hmac.new(secret, msg=request_ctx['data'], digestmod='sha1') secret_comparison = hmac.compare_digest(mac.hexdigest(), signature) # Memory cleanup @@ -73,7 +89,7 @@ def verify_request_secret(request): return secret_comparison -# returns extracted data if it is an event for a PR creation or PR comment creation +# Returns extracted data if it is an event for a PR creation or PR comment creation # if it is a PR comment, only return extracted data if it contains the required keyword specified by the global var # if it doesn't contain the keyword or is not the correct type of event, return None def filter_to_prs_and_pr_comments(json): @@ -85,6 +101,7 @@ def filter_to_prs_and_pr_comments(json): branch_name = json['pull_request']['base']['label'] repo_name = json['repository']['name'] user_name = json['pull_request']['user']['login'] + if branch_name is None: return None @@ -127,59 +144,55 @@ def filter_to_prs_and_pr_comments(json): @app.route(EVENT_URL, methods=['POST']) def init_ci_pipeline(): + request_ctx = filter_to_prs_and_pr_comments(request.json) + if request_ctx is None: + logging.debug("Request filter doesn't match request") + return jsonify({"success": True}) - if not verify_request_ip(request): - print("Incoming webkooh not from a valid Github address") - return jsonify({ - "success": True - }) + request_ctx['src_ip'] = ip_address(u'{}'.format(request.access_route[0])) + request_ctx['X-Hub-Signature'] = request.headers.get('X-Hub-Signature') + request_ctx['data'] = request.data + + if not verify_request_ip(request_ctx): + print("Incoming webhook not from a valid Github address") + log_access_denied(request_ctx, "Incoming webhook not from a valid Github address") + return jsonify({"success": True}) - if not verify_request_secret(request): + if not verify_request_secret(request_ctx): print("Incoming webhook secret doesn't match configured secret") - return jsonify({ - "success": True - }) - - extracted_data = filter_to_prs_and_pr_comments(request.json) - if extracted_data is not None: - if (extracted_data['repo'] == 'openNetVM-dev' or extracted_data['user'] in authorized_users): - print("This is an authorized user") - else: - print("ERROR: This user is not authorized") - os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"User not authorized to run CI, please contact one of the repo maintainers\"" - .format(extracted_data['id'], extracted_data['repo'], extracted_data['body'])) - return jsonify({ - "success": True - }) - - print("Data matches filter, we should RUN CI") - print(extracted_data) - - # Check if there is another CI run in progress - proc1 = subprocess.Popen(['ps', 'cax'], stdout=subprocess.PIPE) - proc2 = subprocess.Popen(['grep', 'manager.sh'], stdin=proc1.stdout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - proc1.stdout.close() - out, err = proc2.communicate() - - if (out): - print("Can't run CI, another CI run in progress") - os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"Another CI run in progress, please try again in 15 minutes\"" - .format(extracted_data['id'], extracted_data['repo'], extracted_data['body'])) - else: - os.system("./manager.sh config {} \"{}\" \"{}\"".format(extracted_data['id'], extracted_data['repo'], extracted_data['body'])) + log_access_denied(request_ctx, "Incoming webhook has an invalid secret") + return jsonify({"success": True}) + + if (request_ctx['repo'] == 'openNetVM' and request_ctx['user'] not in authorized_users): + print("Incoming request is from an unathorized user") + log_access_denied("Incoming request is from an unathorized user") + os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"User not authorized to run CI, please contact one of the repo maintainers\"" + .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) + return jsonify({"success": True}) + + print("Request matches filter, we should RUN CI. {}".format(get_request_info(request_ctx))) + + # Check if there is another CI run in progress + proc1 = subprocess.Popen(['ps', 'cax'], stdout=subprocess.PIPE) + proc2 = subprocess.Popen(['grep', 'manager.sh'], stdin=proc1.stdout, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc1.stdout.close() + out, err = proc2.communicate() + + if (out): + print("Can't run CI, another CI run in progress") + log_access_granted(request_ctx, "CI busy, posting busy msg") + os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"Another CI run in progress, please try again in 15 minutes\"" + .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) else: - print("Data did not match filter, SKIP CI") + log_access_granted(request_ctx, "Running CI") + os.system("./manager.sh config {} \"{}\" \"{}\"".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) - return jsonify({ - "success": True - }) + return jsonify({"status": "ONLINE"}) @app.route("/status", methods=['GET']) def status(): - return jsonify({ - "status": "ONLINE" - }) + return jsonify({"status": "ONLINE"}) if __name__ == "__main__": global KEYWORD @@ -194,9 +207,11 @@ def status(): with open (sys.argv[4], 'r') as cfg: webhook_config = json.load(cfg) + authorized_users = webhook_config['authorized-users'] if authorized_users is None: print("No authroized users found in webhook config") sys.exit(1) + logging.info("Starting the CI service") app.run(host=host, port=port) From 0c259ed6fe4c97ba49e81c826130e38f78f9a01e Mon Sep 17 00:00:00 2001 From: Mykola Date: Wed, 3 Apr 2019 02:07:22 -0400 Subject: [PATCH 6/7] Better logging, optimized encryption --- ci/README.md | 5 ++- ci/webhook-receiver.py | 84 +++++++++++++++++++++++++++++++----------- ci/worker.sh | 2 +- 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/ci/README.md b/ci/README.md index 47e5c0db2..e490ea5d2 100644 --- a/ci/README.md +++ b/ci/README.md @@ -46,7 +46,10 @@ The CI process can be broken into multiple steps: Webhook json config example ``` { - "secret": "look_at_me_i'm_a_secret_key" + "secret-file": "very_special_encrypted_secret_file.bin", + "private-key-file": "private_key.pem", + "log-successful-attempts": true, + "authorized-users": ["puffin", "penguin", "pcoach"] } ``` diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 10b035fb6..fd11ba346 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -14,8 +14,15 @@ import subprocess import logging +# Global vars EVENT_URL = "/github-webhook" -CI_NAME="onvm" +CI_NAME = "onvm" +KEYWORD = None +access_log_enabled = None +authorized_users = None +secret_file_name = None +private_key_file = None +secret = None app = Flask(__name__) @@ -28,14 +35,19 @@ def get_request_info(request_ctx): return "Request details: IP: {}, User: {}, Repo: {}, ID: {}, Body: {}.".format(request_ctx['src_ip'], request_ctx['user'], request_ctx['repo'], request_ctx['id'], request_ctx['body']) def log_access_granted(request_ctx, custom_msg): - logging.info("Access GRANTED: {}. {}".format(custom_msg, get_request_info(request_ctx))) + if (access_log_enabled): + logging.info("Access GRANTED: {}. {}".format(custom_msg, get_request_info(request_ctx))) def log_access_denied(request_ctx, custom_msg): - logging.info("Access DENIED: {}. {}".format(custom_msg, get_request_info(request_ctx))) + logging.warning("Access DENIED: {}. {}".format(custom_msg, get_request_info(request_ctx))) def decrypt_secret(): - secret_file = open(webhook_config['secret-file'], "rb") - private_key = RSA.import_key(open(webhook_config['private-key-file']).read()) + global secret_file_name + global private_key_file_name + global secret + + secret_file = open(secret_file_name, "rb") + private_key = RSA.import_key(open(private_key_file_name).read()) enc_session_key, nonce, tag, ciphertext = \ [ secret_file.read(x) for x in (private_key.size_in_bytes(), 16, 16, -1) ] @@ -49,7 +61,11 @@ def decrypt_secret(): data = cipher_aes.decrypt_and_verify(ciphertext, tag) # Clear memory - secret_file = private_key = enc_session_key = nonce = tag = ciphertext = None + secret_file = private_key = None + private_key_file_name = secret_file_name = None + enc_session_key = nonce = tag = ciphertext = None + del private_key_file_name + del secret_file_name del secret_file del private_key del enc_session_key @@ -61,7 +77,6 @@ def decrypt_secret(): def verify_request_ip(request_ctx): src_ip = request_ctx['src_ip'] - print (src_ip) valid_ips = requests.get('https://api.github.com/meta').json()['hooks'] for ip in valid_ips: @@ -71,20 +86,19 @@ def verify_request_ip(request_ctx): return False def verify_request_secret(request_ctx): + global secret header_signature = request_ctx['X-Hub-Signature'] if header_signature is None: return False signature = header_signature.split('=')[1] - secret = decrypt_secret() mac = hmac.new(secret, msg=request_ctx['data'], digestmod='sha1') secret_comparison = hmac.compare_digest(mac.hexdigest(), signature) # Memory cleanup - header_signature = secret = signature = None + header_signature = signature = None del header_signature - del secret del signature return secret_comparison @@ -97,6 +111,9 @@ def filter_to_prs_and_pr_comments(json): if json is None: return None + if 'action' not in json: + return None + if json['action'] == 'opened' and 'pull_request' in json and 'base' in json['pull_request']: branch_name = json['pull_request']['base']['label'] repo_name = json['repository']['name'] @@ -165,7 +182,7 @@ def init_ci_pipeline(): if (request_ctx['repo'] == 'openNetVM' and request_ctx['user'] not in authorized_users): print("Incoming request is from an unathorized user") - log_access_denied("Incoming request is from an unathorized user") + log_access_denied(request_ctx, "Incoming request is from an unathorized user") os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"User not authorized to run CI, please contact one of the repo maintainers\"" .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) return jsonify({"success": True}) @@ -194,24 +211,49 @@ def init_ci_pipeline(): def status(): return jsonify({"status": "ONLINE"}) -if __name__ == "__main__": - global KEYWORD +def parse_config(cfg_name): + global access_log_enabled + global secret_file_name + global private_key_file_name + global authorized_users - if(len(sys.argv) != 5): - print("Invalid arguments!") + with open (cfg_name, 'r') as cfg: + webhook_config = json.load(cfg) + + access_log_enabled = webhook_config['log-successful-attempts'] + if access_log_enabled is None: + print("Access log switch not specified in the webhook server config") sys.exit(1) - host = sys.argv[1] - port = sys.argv[2] - KEYWORD = sys.argv[3] + secret_file_name = webhook_config['secret-file'] + if secret_file_name is None: + print("No secret file found in the webhook server config") + sys.exit(1) - with open (sys.argv[4], 'r') as cfg: - webhook_config = json.load(cfg) + private_key_file_name = webhook_config['private-key-file'] + if private_key_file_name is None: + print("No private key file found in the webhook server config") + sys.exit(1) authorized_users = webhook_config['authorized-users'] if authorized_users is None: - print("No authroized users found in webhook config") + print("No authroized users found in webhook server config") + sys.exit(1) + + +if __name__ == "__main__": + if(len(sys.argv) != 5): + print("Invalid arguments!") sys.exit(1) + host = sys.argv[1] + port = sys.argv[2] + KEYWORD = sys.argv[3] + cfg_name = sys.argv[4] + + parse_config(cfg_name) + + secret = decrypt_secret() + logging.info("Starting the CI service") app.run(host=host, port=port) diff --git a/ci/worker.sh b/ci/worker.sh index c4f0ada40..ec021a504 100755 --- a/ci/worker.sh +++ b/ci/worker.sh @@ -24,7 +24,7 @@ check_exit_code "ERROR: Building ONVM failed" print_header "Running ONVM Manager" cd onvm -./go.sh 0,1,2 0 0xF0 -s web & +./go.sh 0,1,2,3 0 0xF0 -s web & mgr_pid=$? if [ $mgr_pid -ne 0 ] then From 8526e7693020876daa306d423ac2cf7472bc1526 Mon Sep 17 00:00:00 2001 From: Mykola Date: Wed, 3 Apr 2019 14:00:18 -0400 Subject: [PATCH 7/7] Fix unused var --- ci/webhook-receiver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index fd11ba346..e6230b6e4 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -44,7 +44,6 @@ def log_access_denied(request_ctx, custom_msg): def decrypt_secret(): global secret_file_name global private_key_file_name - global secret secret_file = open(secret_file_name, "rb") private_key = RSA.import_key(open(private_key_file_name).read())