From 91b30f2ba34e368266bd60920bcd8f0413185989 Mon Sep 17 00:00:00 2001 From: maxklema Date: Thu, 3 Jul 2025 00:16:21 +0000 Subject: [PATCH 01/25] (generous) retry quota on all inputs --- .../get-lxc-container-details.sh | 77 ++++++++++++++----- 1 file changed, 56 insertions(+), 21 deletions(-) diff --git a/container-creation/get-lxc-container-details.sh b/container-creation/get-lxc-container-details.sh index 9715de8c..0cf88954 100644 --- a/container-creation/get-lxc-container-details.sh +++ b/container-creation/get-lxc-container-details.sh @@ -37,7 +37,7 @@ while [ $USER_AUTHENTICATED == 'false' ]; do RETRIES=$(($RETRIES-1)) else echo "Too many incorrect attempts. Exiting..." - exit 0 + exit 2 fi done @@ -50,16 +50,24 @@ if [ -z "$CONTAINER_NAME" ]; then fi HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") +HOST_NAME_RETRIES=10 while [ $HOST_NAME_EXISTS == 'true' ]; do - echo "Sorry! That name has already been registered. Try another name" - read -p "Enter Application Name (One-Word) → " CONTAINER_NAME - HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") + if [ $HOST_NAME_RETRIES -gt 0 ]; then + echo "Sorry! That name has already been registered. Try another name" + read -p "Enter Application Name (One-Word) → " CONTAINER_NAME + HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") + HOST_NAME_RETRIES=$(($HOST_NAME_RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 3 + fi done echo "✅ $CONTAINER_NAME is available" # Gather Container Password +PASSWORD_RETRIES=10 if [ -z "$CONTAINER_PASSWORD" ]; then read -sp "Enter Container Password → " CONTAINER_PASSWORD @@ -68,19 +76,32 @@ if [ -z "$CONTAINER_PASSWORD" ]; then echo while [[ "$CONFIRM_PASSWORD" != "$CONTAINER_PASSWORD" || ${#CONTAINER_PASSWORD} -lt 8 ]]; do - echo "Sorry, try again. Ensure passwords are at least 8 characters." - read -sp "Enter Container Password → " CONTAINER_PASSWORD - echo - read -sp "Confirm Container Password → " CONFIRM_PASSWORD - echo + if [ $PASSWORD_RETRIES -gt 0 ]; then + echo "Sorry, try again. Ensure passwords are at least 8 characters." + read -sp "Enter Container Password → " CONTAINER_PASSWORD + echo + read -sp "Confirm Container Password → " CONFIRM_PASSWORD + echo + PASSWORD_RETRIES=$(($PASSWORD_RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 4 + fi done else - while [ ${#CONTAINER_PASSWORD} -lt 8 ]; do - echo "Sorry, try again. Ensure passwords are at least 8 characters." - read -sp "Enter Container Password → " CONTAINER_PASSWORD - echo - read -sp "Confirm Container Password → " CONFIRM_PASSWORD - echo + CONFIRM_PASSWORD="$CONTAINER_PASSWORD" + while [[ "$CONFIRM_PASSWORD" != "$CONTAINER_PASSWORD" || ${#CONTAINER_PASSWORD} -lt 8 ]]; do + if [ $PASSWORD_RETRIES -gt 0 ]; then + echo "Sorry, try again. Ensure passwords are at least 8 characters." + read -sp "Enter Container Password → " CONTAINER_PASSWORD + echo + read -sp "Confirm Container Password → " CONFIRM_PASSWORD + echo + PASSWORD_RETRIES=$(($PASSWORD_RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 4 + fi done fi @@ -94,6 +115,7 @@ PUB_FILE="key_$RANDOM_NUM.pub" TEMP_PUB_FILE="/root/bin/ssh/temp_pubs/$PUB_FILE" # in case two users are running this script at the same time, they do not overwrite each other's temp files touch "$TEMP_PUB_FILE" DETECT_PUBLIC_KEY=$(sudo /root/bin/ssh/detectPublicKey.sh "$SSH_KEY_FP" "$TEMP_PUB_FILE") +KEY_RETRIES=10 if [ "$DETECT_PUBLIC_KEY" == "Public key found for create-container" ]; then echo "🔐 Public Key Found!" @@ -107,8 +129,14 @@ else # Check if key is valid while [[ "$PUBLIC_KEY" != "" && $(echo "$PUBLIC_KEY" | ssh-keygen -l -f - 2>&1 | tr -d '\r') == "(stdin) is not a public key file." ]]; do - echo "❌ \"$PUBLIC_KEY\" is not a valid key. Enter either a valid key or leave blank to skip." - read -p "Enter Public Key (Allows Easy Access to Container) [OPTIONAL - LEAVE BLANK TO SKIP] → " PUBLIC_KEY + if [ $KEY_RETRIES -gt 0 ]; then + echo "❌ \"$PUBLIC_KEY\" is not a valid key. Enter either a valid key or leave blank to skip." + read -p "Enter Public Key (Allows Easy Access to Container) [OPTIONAL - LEAVE BLANK TO SKIP] → " PUBLIC_KEY + KEY_RETRIES=$(($KEY_RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 5 + fi done if [ "$PUBLIC_KEY" != "" ]; then @@ -119,14 +147,21 @@ else fi # Get HTTP Port Container Listens On +HTTP_PORT_RETRIES=10 if [ -z "$HTTP_PORT" ]; then read -p "Enter HTTP Port for your container to listen on (80-9999) → " HTTP_PORT fi while ! [[ "$HTTP_PORT" =~ ^[0-9]+$ ]] || [ "$HTTP_PORT" -lt 80 ] || [ "$HTTP_PORT" -gt 9999 ]; do - echo "❌ Invalid HTTP Port. It must be a number between 80 and 9,999." - read -p "Enter HTTP Port for your container to listen on (80-9999) → " HTTP_PORT + if [ $HTTP_PORT_RETRIES -gt 0 ]; then + echo "❌ Invalid HTTP Port. It must be a number between 80 and 9,999." + read -p "Enter HTTP Port for your container to listen on (80-9999) → " HTTP_PORT + HTTP_PORT_RETRIES=$(($HTTP_PORT_RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 6 + fi done echo "✅ HTTP Port is set to $HTTP_PORT" @@ -147,7 +182,7 @@ protocol_duplicate() { } read -p "Does your Container require any protocols other than SSH and HTTP? (y/n) → " USE_OTHER_PROTOCOLS -while [ "${USE_OTHER_PROTOCOLS^^}" != "Y" ] && [ "${USE_OTHER_PROTOCOLS^^}" != "N" ]; do +while [ "${USE_OTHER_PROTOCOLS^^}" != "Y" ] && [ "${USE_OTHER_PROTOCOLS^^}" != "N" ] && [ "${USER_OTHER_PROTOCOLS^^}" != "" ]; do echo "Please answer 'y' for yes or 'n' for no." read -p "Does your Container require any protocols other than SSH and HTTP? (y/n) → " USE_OTHER_PROTOCOLS done @@ -215,4 +250,4 @@ rm -rf "$TEMP_PUB_FILE" unset CONFIRM_PASSWORD unset CONTAINER_PASSWORD -unset PUBLIC_KEY \ No newline at end of file +unset PUBLIC_KEY From 9490abfb108c0d2b44a673a3fc6482fbfc97a8c1 Mon Sep 17 00:00:00 2001 From: maxklema Date: Thu, 3 Jul 2025 15:02:08 +0000 Subject: [PATCH 02/25] beginning deployment script --- container-creation/create-container.sh | 4 + container-creation/deploy-application.sh | 77 +++++++++++++++++++ .../get-lxc-container-details.sh | 5 +- 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100755 container-creation/deploy-application.sh diff --git a/container-creation/create-container.sh b/container-creation/create-container.sh index 583be02a..8703bc2d 100644 --- a/container-creation/create-container.sh +++ b/container-creation/create-container.sh @@ -99,6 +99,10 @@ MAGENTA='\033[35m' GREEN='\033[32m' RESET='\033[0m' +if (( $NEXT_ID % 2 == 0 )); then + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +fi + echo -e "📦 ${BLUE}Container ID :${RESET} $NEXT_ID" echo -e "🌐 ${MAGENTA}Internal IP :${RESET} $CONTAINER_IP" echo -e "🔗 ${GREEN}Domain Name :${RESET} https://$CONTAINER_NAME.opensource.mieweb.org" diff --git a/container-creation/deploy-application.sh b/container-creation/deploy-application.sh new file mode 100755 index 00000000..43bb9f6c --- /dev/null +++ b/container-creation/deploy-application.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Helper script to gather project details for automatic deployment +# Modified July 3rd, 2025 by Maxwell Klema +# ------------------------------------------ + +# Define color variables (works on both light and dark backgrounds) +RESET="\033[0m" +BOLD="\033[1m" +MAGENTA='\033[35m' + +echo -e "${BOLD}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +echo -e "${BOLD}${MAGENTA}🌐 Let's Get Your Project Automatically Deployed ${RESET}" +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n${RESET}" + +# Get and validate project repository + +if [ -z "$PROJECT_REPOSITORY" ]; then + read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY +fi + + +while ! git ls-remote --heads "$PROJECT_REPOSITORY" > /dev/null 2>&1 ; do + echo "⚠️ The repository link you provided, \"$PROJECT_REPOSITORY\" was not valid." + read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY +done + +# Get Repository Branch + +if [ -z "$PROJECT_BRANCH" ]; then + read -p "🪾 Enter the project branch to deploy from (leave blank for main) → " PROJECT_BRANCH +fi + +if [ "$PROJECT_BRANCH" == "" ]; then + PROJECT_BRANCH="main" +fi + +while ! git ls-remote --heads "$PROJECT_REPOSITORY" | grep -q "refs/heads/$PROJECT_BRANCH"; do + echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." + read -p "🪾 Enter the project branch to deploy from (leave blank for main) → " PROJECT_BRANCH +done + +# Get Project Root Directory + +if [ -z "$PROJECT_ROOT" ]; then + read -p "📁 Enter the project root directory (relative to repo root, or '.' for root) → " PROJECT_ROOT +fi + +# Get Environment Variables +if [ -z "$ENV_VARS" ]; then + read -p "🔑 Enter any environment variables (KEY=VALUE, comma separated, leave blank if none) → " ENV_VARS +fi + +# Get Install Command +if [ -z "$INSTALL_COMMAND" ]; then + read -p "📦 Enter the install command (e.g., 'npm install', 'pip install') → " INSTALL_COMMAND +fi + +# Get Build Command +if [ -z "$BUILD_COMMAND" ]; then + read -p "🏗️ Enter the build command (leave blank if not needed) → " BUILD_COMMAND +fi + +# Get Output Directory +if [ -z "$OUTPUT_DIRECTORY" ]; then + read -p "📂 Enter the output directory (e.g., 'dist', 'build', leave blank if not applicable) → " OUTPUT_DIRECTORY +fi + +# Get Start Command +if [ -z "$START_COMMAND" ]; then + read -p "🚦 Enter the start command (e.g., 'npm start', 'python app.py') → " START_COMMAND +fi + +# Get Runtime Language +if [ -z "$RUNTIME_LANGUAGE" ]; then + read -p "🖥️ Enter the runtime language (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE +fi + diff --git a/container-creation/get-lxc-container-details.sh b/container-creation/get-lxc-container-details.sh index 0cf88954..0f3def64 100644 --- a/container-creation/get-lxc-container-details.sh +++ b/container-creation/get-lxc-container-details.sh @@ -49,12 +49,13 @@ if [ -z "$CONTAINER_NAME" ]; then read -p "Enter Application Name (One-Word) → " CONTAINER_NAME fi +CONTAINER_NAME="${CONTAINER_NAME,,}" #convert to lowercase HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") HOST_NAME_RETRIES=10 -while [ $HOST_NAME_EXISTS == 'true' ]; do +while [[ $HOST_NAME_EXISTS == 'true' ]] || [[ ! "$CONTAINER_NAME" =~ ^[A-Za-z0-9-]+$ ]]; do if [ $HOST_NAME_RETRIES -gt 0 ]; then - echo "Sorry! That name has already been registered. Try another name" + echo "Sorry! Either that name has already been registered or your hostname is ill-formatted. Try another name" read -p "Enter Application Name (One-Word) → " CONTAINER_NAME HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") HOST_NAME_RETRIES=$(($HOST_NAME_RETRIES-1)) From b832d5b37ac680f802c64b980ef68ac847155944 Mon Sep 17 00:00:00 2001 From: maxklema Date: Mon, 7 Jul 2025 20:02:52 +0000 Subject: [PATCH 03/25] Automated Deployment Script - Gathering Repo Details --- container-creation/deploy-application.sh | 67 +++++++++++++++++-- container-creation/js/authenticateRepo.js | 23 +++++++ container-creation/js/authenticateUser.js | 4 +- .../{authenticateUserRunner.js => runner.js} | 8 ++- 4 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 container-creation/js/authenticateRepo.js rename container-creation/js/{authenticateUserRunner.js => runner.js} (54%) diff --git a/container-creation/deploy-application.sh b/container-creation/deploy-application.sh index 43bb9f6c..69811712 100755 --- a/container-creation/deploy-application.sh +++ b/container-creation/deploy-application.sh @@ -27,7 +27,7 @@ done # Get Repository Branch if [ -z "$PROJECT_BRANCH" ]; then - read -p "🪾 Enter the project branch to deploy from (leave blank for main) → " PROJECT_BRANCH + read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH fi if [ "$PROJECT_BRANCH" == "" ]; then @@ -36,20 +36,77 @@ fi while ! git ls-remote --heads "$PROJECT_REPOSITORY" | grep -q "refs/heads/$PROJECT_BRANCH"; do echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." - read -p "🪾 Enter the project branch to deploy from (leave blank for main) → " PROJECT_BRANCH + read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH done # Get Project Root Directory if [ -z "$PROJECT_ROOT" ]; then - read -p "📁 Enter the project root directory (relative to repo root, or '.' for root) → " PROJECT_ROOT + read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT fi +if [ "$PROJECT_ROOT" == "" ]; then + PROJECT_ROOT="/" +fi + +VALID_PROJECT_ROOT=$(node js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") + +while [ "$VALID_PROJECT_ROOT" == "false" ]; do + echo "⚠️ The root directory you provided, \"$PROJECT_ROOT\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." + read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT + VALID_PROJECT_ROOT=$(node js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") +done + # Get Environment Variables -if [ -z "$ENV_VARS" ]; then - read -p "🔑 Enter any environment variables (KEY=VALUE, comma separated, leave blank if none) → " ENV_VARS + +gatherEnvVars(){ + + read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY + read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE + + while [ "$ENV_VAR_KEY" == "" ] || [ "$ENV_VAR_VALUE" == "" ]; do + echo "⚠️ Key or value cannot be empty. Try again." + read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY + read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE + done + + echo "$ENV_VAR_KEY=$ENV_VAR_VALUE" >> $TEMP_ENV_FILE_PATH + + read -p "🔑 Do you want to enter another Environment Variable? (y/n) → " ENTER_ANOTHER_ENV +} + +if [ -z "$REQUIRE_ENV_VARS" ]; then + read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS +fi + +while [ "${REQUIRE_ENV_VARS^^}" != "Y" ] && [ "${REQUIRE_ENV_VARS^^}" != "N" ] && [ "${REQUIRE_ENV_VARS^^}" != "" ]; do + echo "⚠️ Invalid option. Please try again." + read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS +done + +if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then + + # generate random temp .env file + RANDOM_NUM=$(shuf -i 100000-999999 -n 1) + ENV_FILE="env_$RANDOM_NUM.txt" + TEMP_ENV_FILE_PATH="/root/bin/env/$ENV_FILE" + touch "$TEMP_ENV_FILE_PATH" + + if [ ! -z "$CONTAINER_ENV_VARS" ]; then + echo "$CONTAINER_ENV_VARS " | jq -r 'to_entries[] | "\(.key)=\(.value)"' > "$TEMP_ENV_FILE_PATH" #k=v pairs + else + gatherEnvVars + while [ "${ENTER_ANOTHER_ENV^^}" == "Y" ]; do + gatherEnvVars + done + fi fi + + + + + # Get Install Command if [ -z "$INSTALL_COMMAND" ]; then read -p "📦 Enter the install command (e.g., 'npm install', 'pip install') → " INSTALL_COMMAND diff --git a/container-creation/js/authenticateRepo.js b/container-creation/js/authenticateRepo.js new file mode 100644 index 00000000..37a57e7a --- /dev/null +++ b/container-creation/js/authenticateRepo.js @@ -0,0 +1,23 @@ +const axios = require('axios'); + +function authenticateRepo(repositoryURL, branch, folderPath) { + + if (folderPath.indexOf('.') != -1) { + return Promise.resolve(false); //early exit if path points to a specific file + } + + repositoryURL = repositoryURL.replace('.git', ''); + if (folderPath.startsWith('/')) { + folderPath = folderPath.substring(1, folderPath.length); + } + fullURL = `${repositoryURL}/tree/${branch}/${folderPath}` + + const config = { + method: "get", + url: fullURL + } + + return axios.request(config).then((response) => response.status === 200).catch(() => false); +} + +module.exports = { authenticateRepo } \ No newline at end of file diff --git a/container-creation/js/authenticateUser.js b/container-creation/js/authenticateUser.js index 61153f1e..36942c4f 100644 --- a/container-creation/js/authenticateUser.js +++ b/container-creation/js/authenticateUser.js @@ -1,6 +1,3 @@ -// Script to authenticate a user into Proxmox -// Last updated June 24th, 2025 by Maxwell Klema - const axios = require('axios'); const qs = require('qs'); const https = require('https'); @@ -27,4 +24,5 @@ function authenticateUser(username, password) { return axios.request(config).then((response) => response.status === 200).catch(() => false); } + module.exports = { authenticateUser }; diff --git a/container-creation/js/authenticateUserRunner.js b/container-creation/js/runner.js similarity index 54% rename from container-creation/js/authenticateUserRunner.js rename to container-creation/js/runner.js index 5843dd9c..8fde30cb 100644 --- a/container-creation/js/authenticateUserRunner.js +++ b/container-creation/js/runner.js @@ -1,11 +1,13 @@ -// Script to run authenticateUser in the shell -// Last updated June 24th, 2025 by Maxwell Klema - authenticateuser = require("./authenticateUser.js"); +authenticaterepo = require("./authenticateRepo.js") const [, , func, ...args] = process.argv; if (func == "authenticateUser") { authenticateuser.authenticateUser(...args).then((result) => { console.log(result); }); +} else if (func == "authenticateRepo") { + authenticaterepo.authenticateRepo(...args).then((result) => { + console.log(result); + }) } From ef6e329f5f3fc78771d67dccf5050b79fd2cc9c2 Mon Sep 17 00:00:00 2001 From: maxklema Date: Tue, 8 Jul 2025 21:08:51 +0000 Subject: [PATCH 04/25] changes to deployment script - writing service installation commands --- container-creation/deploy-application.sh | 129 ++++++++++++++++--- container-creation/services/service_map.json | 69 ++++++++++ 2 files changed, 179 insertions(+), 19 deletions(-) create mode 100644 container-creation/services/service_map.json diff --git a/container-creation/deploy-application.sh b/container-creation/deploy-application.sh index 69811712..e2bd21f8 100755 --- a/container-creation/deploy-application.sh +++ b/container-creation/deploy-application.sh @@ -12,7 +12,7 @@ echo -e "${BOLD}\n━━━━━━━━━━━━━━━━━━━━ echo -e "${BOLD}${MAGENTA}🌐 Let's Get Your Project Automatically Deployed ${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n${RESET}" -# Get and validate project repository +# Get and validate project repository ======== if [ -z "$PROJECT_REPOSITORY" ]; then read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY @@ -24,7 +24,7 @@ while ! git ls-remote --heads "$PROJECT_REPOSITORY" > /dev/null 2>&1 ; do read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY done -# Get Repository Branch +# Get Repository Branch ======== if [ -z "$PROJECT_BRANCH" ]; then read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH @@ -39,7 +39,7 @@ while ! git ls-remote --heads "$PROJECT_REPOSITORY" | grep -q "refs/heads/$PROJE read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH done -# Get Project Root Directory +# Get Project Root Directory ======== if [ -z "$PROJECT_ROOT" ]; then read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT @@ -57,7 +57,7 @@ while [ "$VALID_PROJECT_ROOT" == "false" ]; do VALID_PROJECT_ROOT=$(node js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") done -# Get Environment Variables +# Get Environment Variables ======== gatherEnvVars(){ @@ -93,7 +93,12 @@ if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then touch "$TEMP_ENV_FILE_PATH" if [ ! -z "$CONTAINER_ENV_VARS" ]; then - echo "$CONTAINER_ENV_VARS " | jq -r 'to_entries[] | "\(.key)=\(.value)"' > "$TEMP_ENV_FILE_PATH" #k=v pairs + if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success + echo "$CONTAINER_ENV_VARS " | jq -r 'to_entries[] | "\(.key)=\(.value)"' > "$TEMP_ENV_FILE_PATH" #k=v pairs + else + echo "⚠️ Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." + exit 10 + fi else gatherEnvVars while [ "${ENTER_ANOTHER_ENV^^}" == "Y" ]; do @@ -102,33 +107,119 @@ if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then fi fi +# Get Install Command ======== - - - - -# Get Install Command if [ -z "$INSTALL_COMMAND" ]; then - read -p "📦 Enter the install command (e.g., 'npm install', 'pip install') → " INSTALL_COMMAND + read -p "📦 Enter the install command (e.g., 'npm install') → " INSTALL_COMMAND fi -# Get Build Command +while [ "$INSTALL_COMMAND" == "" ]; do + echo "⚠️ The install command cannot be blank. Please try again." + read -p "📦 Enter the install command (e.g., 'npm install') → " INSTALL_COMMAND +done + +# Get Build Command ======== + if [ -z "$BUILD_COMMAND" ]; then - read -p "🏗️ Enter the build command (leave blank if not needed) → " BUILD_COMMAND + read -p "🏗️ Enter the build command (leave blank if no build command) → " BUILD_COMMAND fi -# Get Output Directory -if [ -z "$OUTPUT_DIRECTORY" ]; then - read -p "📂 Enter the output directory (e.g., 'dist', 'build', leave blank if not applicable) → " OUTPUT_DIRECTORY +# Get Build Directory ======== + +if [ -z "$BUILD_DIRECTORY" ]; then + read -p "📂 Enter the build directory (e.g., 'dist'; leave blank if not applicable) → " BUILD_DIRECTORY fi -# Get Start Command +while [ "$BUILD_COMMAND" == "" ] && [ "$BUILD_DIRECTORY" != "" ]; do + echo "⚠️ You did not enter a build command. The build directory should be empty, too. Please try again." + read -p "📂 Enter the build directory (e.g., 'dist', 'build', leave blank if not applicable) → " BUILD_DIRECTORY +done + +# Get Start Command ======== + if [ -z "$START_COMMAND" ]; then read -p "🚦 Enter the start command (e.g., 'npm start', 'python app.py') → " START_COMMAND fi -# Get Runtime Language +while [ "$START_COMMAND" == "" ]; do + echo "⚠️ The start command cannot be blank. Please try again." + read -p "🚦 Enter the start command (e.g., 'npm start') → " START_COMMAND +done + +# Get Runtime Language ======== + if [ -z "$RUNTIME_LANGUAGE" ]; then - read -p "🖥️ Enter the runtime language (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE + read -p "🖥️ Enter the underlying runtime environment (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE +fi + +while [ "${RUNTIME_LANGUAGE^^}" != "NODEJS" ] && [ "${RUNTIME_LANGUAGE^^}" != "PYTHON" ]; do + echo "⚠️ Sorry, that runtime environment is not yet supported. Only \"nodejs\" and \"python\" are currently supported." + read -p "🖥️ Enter the underlying runtime environment (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE +done + +# Get Services ======== + +SERVICE_MAP="services/service_map.json" + +# Helper function to check if a user has added the same service twice +serviceExists() { + SERVICE="$1" + APPENDED_SERVICES="$2" + + for CURRENT in "${APPENDED_SERVICES[@]}"; do + if [ "${SERVICE,,}" == "${CURRENT,,}" ]; then + return 0 + fi + done + return 1 +} + +# Helper function to gather a service name or other valid option +gatherService() { + read -p "➡️ Enter the name of a service to add to your container or type \"C\" to create a custom service (\"E\" to exit) → " SERVICE + while [ "$SERVICE" == "" ]; do + echo "⚠️ Invalid option. Please try again." + read -p "➡️ Enter the name of a service to add to your container or type \"C\" to create a custom service (\"E\" to exit) → " SERVICE + done +} + +# Helper function to append a new service to a container +appendService() { + gatherService + + APPENDED_SERVICES=() + SERVICE_IN_MAP=$(jq -r --arg key "${SERVICE,,}" '.[$key] // empty' "$SERVICE_MAP") + + #Check if service is in services/service_map.json (not null) + if ! serviceExists "$SERVICE" "$APPENDED_SERVICES" && [ "${SERVICE^^}" != "C" ] && [ "${SERVICE^^}" != "E" ] && [ -n "$SERVICE_IN_MAP" ]; then + jq -r --arg key "${SERVICE,,}" '.[$key][]' "$SERVICE_MAP" >> "$TEMP_SERVICES_FILE_PATH" + echo "sudo systemctl daemon-reload" >> "$TEMP_SERVICES_FILE_PATH" + echo "✅ $SERVICE added to your container." + APPENDED_SERVICES+=("${SERVICE^^}") + fi +} + +if [ -z "$REQUIRE_SERVICES" ]; then + read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES +fi + +while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do + echo "⚠️ Invalid option. Please try again." + read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES +done + +if [ "${REQUIRE_SERVICES^^}" == "Y" ]; then + + # Generate random (temporary) file to store install commands for needed services + RANDOM_NUM=$(shuf -i 100000-999999 -n 1) + SERVICES_FILE="services_$RANDOM_NUM.txt" + TEMP_SERVICES_FILE_PATH="/root/bin/services/$SERVICES_FILE" + touch "$TEMP_SERVICES_FILE_PATH" + + appendService + while [ "${SERVICE^^}" != "E" ]; do + appendService + done fi + diff --git a/container-creation/services/service_map.json b/container-creation/services/service_map.json new file mode 100644 index 00000000..8f7618f4 --- /dev/null +++ b/container-creation/services/service_map.json @@ -0,0 +1,69 @@ +{ + "meteor": [ + "curl https://install.meteor.com/ | sh" + ], + "mongodb": [ + "sudo apt-get update", + "sudo apt-get install -y gnupg curl", + "curl -fsSL https://pgp.mongodb.com/server-7.0.asc | sudo gpg --dearmor -o /usr/share/keyrings/mongodb-server-7.0.gpg", + "echo \"deb [ signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/debian bookworm/mongodb-org/7.0 main\" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list", + "sudo apt-get update", + "sudo apt-get install -y mongodb-org", + "sudo systemctl enable mongod", + "sudo systemctl start mongod" + ], + "redis": [ + "sudo apt-get update", + "sudo apt-get install -y redis-server", + "sudo systemctl enable redis-server", + "sudo systemctl start redis-server" + ], + "postgresql": [ + "sudo apt-get update", + "sudo apt-get install -y postgresql postgresql-contrib", + "sudo systemctl enable postgresql", + "sudo systemctl start postgresql" + ], + "apache": [ + "sudo apt-get update", + "sudo apt-get install -y apache2", + "sudo systemctl enable apache2", + "sudo systemctl start apache2" + ], + "nginx": [ + "sudo apt-get update", + "sudo apt-get install -y nginx", + "sudo systemctl enable nginx", + "sudo systemctl start nginx" + ], + "docker": [ + "sudo apt-get update", + "sudo apt-get install -y lsb-release", + "sudo apt-get install -y ca-certificates curl gnupg lsb-release", + "sudo install -m 0755 -d /etc/apt/keyrings", + "curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg", + "echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable\" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null", + "sudo apt-get update", + "sudo apt-get install -y docker-ce docker-ce-cli containerd.io", + "sudo systemctl enable docker", + "sudo systemctl start docker" + ], + "rabbitmq": [ + "sudo apt-get update", + "sudo apt-get install -y rabbitmq-server", + "sudo systemctl enable rabbitmq-server", + "sudo systemctl start rabbitmq-server" + ], + "memcached": [ + "sudo apt-get update", + "sudo apt-get install -y memcached", + "sudo systemctl enable memcached", + "sudo systemctl start memcached" + ], + "mariadb": [ + "sudo apt-get update", + "sudo apt-get install -y mariadb-server", + "sudo systemctl enable mariadb", + "sudo systemctl start mariadb" + ] +} \ No newline at end of file From aadd883fd43c7442f995d0a08fb1898e9c079499 Mon Sep 17 00:00:00 2001 From: maxklema Date: Wed, 9 Jul 2025 18:21:52 +0000 Subject: [PATCH 05/25] Services Portion of Deployment Script Finished. Custom Environment Variable Logic for Services and Custom Services --- container-creation/deploy-application.sh | 106 ++++++++++++++++++----- 1 file changed, 84 insertions(+), 22 deletions(-) diff --git a/container-creation/deploy-application.sh b/container-creation/deploy-application.sh index e2bd21f8..81682486 100755 --- a/container-creation/deploy-application.sh +++ b/container-creation/deploy-application.sh @@ -1,6 +1,6 @@ #!/bin/bash # Helper script to gather project details for automatic deployment -# Modified July 3rd, 2025 by Maxwell Klema +# Modified July 9th, 2025 by Maxwell Klema # ------------------------------------------ # Define color variables (works on both light and dark backgrounds) @@ -160,12 +160,11 @@ done # Get Services ======== SERVICE_MAP="services/service_map.json" +APPENDED_SERVICES=() # Helper function to check if a user has added the same service twice serviceExists() { SERVICE="$1" - APPENDED_SERVICES="$2" - for CURRENT in "${APPENDED_SERVICES[@]}"; do if [ "${SERVICE,,}" == "${CURRENT,,}" ]; then return 0 @@ -174,31 +173,81 @@ serviceExists() { return 1 } -# Helper function to gather a service name or other valid option -gatherService() { - read -p "➡️ Enter the name of a service to add to your container or type \"C\" to create a custom service (\"E\" to exit) → " SERVICE - while [ "$SERVICE" == "" ]; do - echo "⚠️ Invalid option. Please try again." - read -p "➡️ Enter the name of a service to add to your container or type \"C\" to create a custom service (\"E\" to exit) → " SERVICE - done +processService() { + local SERVICE="$1" + local MODE="$2" # "batch" or "single" + + SERVICE_IN_MAP=$(jq -r --arg key "${SERVICE,,}" '.[$key] // empty' "$SERVICE_MAP") + if serviceExists "$SERVICE"; then + if [ "$MODE" = "batch" ]; then + return 0 # skip to next in batch mode + else + echo "⚠️ You already added \"$SERVICE\" as a service. Please try again." + return 0 + fi + elif [ "${SERVICE^^}" != "C" ] && [ "${SERVICE^^}" != "" ] && [ -n "$SERVICE_IN_MAP" ]; then + jq -r --arg key "${SERVICE,,}" '.[$key][]' "$SERVICE_MAP" >> "$TEMP_SERVICES_FILE_PATH" + echo "sudo systemctl daemon-reload" >> "$TEMP_SERVICES_FILE_PATH" + echo "✅ ${SERVICE^^} added to your container." + APPENDED_SERVICES+=("${SERVICE^^}") + elif [ "${SERVICE^^}" == "C" ]; then + appendCustomService + elif [ "${SERVICE^^}" != "" ]; then + echo "⚠️ Service \"$SERVICE\" does not exist." + [ "$MODE" = "batch" ] && exit 20 + fi } # Helper function to append a new service to a container appendService() { - gatherService + if [ ! -z "$SERVICES" ]; then + for SERVICE in $(echo "$SERVICES" | jq -r '.[]'); do + processService "$SERVICE" "batch" + done + else + read -p "➡️ Enter the name of a service to add to your container or type \"C\" to set up a custom service installation (Enter to exit) → " SERVICE + processService "$SERVICE" "single" + fi +} - APPENDED_SERVICES=() - SERVICE_IN_MAP=$(jq -r --arg key "${SERVICE,,}" '.[$key] // empty' "$SERVICE_MAP") +appendCustomService() { + # If there is an env variable for custom services, iterate through each command and append it to temporary services file + if [ ! -z "$CUSTOM_SERVICES" ]; then + echo "$CUSTOM_SERVICES" | jq -c -r '.[]' | while read -r CUSTOM_SERVICE; do + echo "$CUSTOM_SERVICE" | jq -c -r '.[]' | while read -r CUSTOM_SERVICE_COMMAND; do + if [ ! -z "$CUSTOM_SERVICE_COMMAND" ]; then + echo "$CUSTOM_SERVICE_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" + else + echo "⚠️ Command cannot be empty." + exit 21; + fi + done + done + echo "✅ Custom Services appended." + else + echo "🛎️ Configuring Custom Service Installation. For each prompt, enter a command that is a part of the installation process for your service on Debian Bookworm. Do not forget to enable and start the service at the end. Once you have entered all of your commands, press enter to continue" + COMMAND_NUM=1 + read -p "➡️ Enter Command $COMMAND_NUM: " CUSTOM_COMMAND - #Check if service is in services/service_map.json (not null) - if ! serviceExists "$SERVICE" "$APPENDED_SERVICES" && [ "${SERVICE^^}" != "C" ] && [ "${SERVICE^^}" != "E" ] && [ -n "$SERVICE_IN_MAP" ]; then - jq -r --arg key "${SERVICE,,}" '.[$key][]' "$SERVICE_MAP" >> "$TEMP_SERVICES_FILE_PATH" - echo "sudo systemctl daemon-reload" >> "$TEMP_SERVICES_FILE_PATH" - echo "✅ $SERVICE added to your container." - APPENDED_SERVICES+=("${SERVICE^^}") + echo "$CUSTOM_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" + + while [ "${CUSTOM_COMMAND^^}" != "" ]; do + ((COMMAND_NUM++)) + read -p "➡️ Enter Command $COMMAND_NUM: " CUSTOM_COMMAND + echo "$CUSTOM_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" + done fi } +# Helper function to see if a user wants to set up a custom service +setUpService() { + read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION + while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do + echo "⚠️ Invalid option. Please try again." + read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION + done +} + if [ -z "$REQUIRE_SERVICES" ]; then read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES fi @@ -217,9 +266,22 @@ if [ "${REQUIRE_SERVICES^^}" == "Y" ]; then touch "$TEMP_SERVICES_FILE_PATH" appendService - while [ "${SERVICE^^}" != "E" ]; do - appendService + while [ "${SERVICE^^}" != "" ] || [ ! -z "$SERVICES" ]; do + if [ -z "$SERVICES" ]; then + appendService + else + if [ ! -z "$CUSTOM_SERVICES" ]; then # assumes both services and custom services passed as ENV vars + appendCustomService + else # custom services not passed as ENV var, so must prompt the user for their custom services + setUpService + while [ "${SETUP_CUSTOM_SERVICE_INSTALLATION^^}" == "Y" ]; do + appendCustomService + setUpService + done + fi + break + fi done fi - +echo -e "\n✅ Deployment Process Finished.\n" From c54ca7e74b21257ef26cce8b9a7f63ecfe906495 Mon Sep 17 00:00:00 2001 From: maxklema Date: Thu, 10 Jul 2025 00:46:59 +0000 Subject: [PATCH 06/25] Prototyping Automatic Deployment Script --- container creation/automaticLXCDeployment.sh | 46 +++++++++++++++++++ .../create-container.sh | 0 .../get-deployment-details.sh | 0 .../get-lxc-container-details.sh | 0 .../js/authenticateRepo.js | 0 .../js/authenticateUser.js | 0 .../js/runner.js | 0 .../services/service_map.json | 0 .../ssh/detectPublicKey.sh | 0 .../ssh/publicKeyAppendJumpHost.sh | 0 .../register-container.sh | 0 .../register_proxy_hook.sh | 0 .../dnsmasq.conf | 0 .../extract-fingerprint.sh | 0 {intern-nginx => nginx proxy}/nginx.conf | 0 {intern-nginx => nginx proxy}/port_map.js | 0 .../reverse_proxy.conf | 0 17 files changed, 46 insertions(+) create mode 100644 container creation/automaticLXCDeployment.sh rename {container-creation => container creation}/create-container.sh (100%) rename container-creation/deploy-application.sh => container creation/get-deployment-details.sh (100%) rename {container-creation => container creation}/get-lxc-container-details.sh (100%) rename {container-creation => container creation}/js/authenticateRepo.js (100%) rename {container-creation => container creation}/js/authenticateUser.js (100%) rename {container-creation => container creation}/js/runner.js (100%) rename {container-creation => container creation}/services/service_map.json (100%) rename {container-creation => container creation}/ssh/detectPublicKey.sh (100%) rename {container-creation => container creation}/ssh/publicKeyAppendJumpHost.sh (100%) rename {intern-phxdc-pve1 => container registration}/register-container.sh (100%) rename {intern-phxdc-pve1 => container registration}/register_proxy_hook.sh (100%) rename {intern-dnsmasq => dnsmasq service}/dnsmasq.conf (100%) rename {intern-jump => jump server}/extract-fingerprint.sh (100%) rename {intern-nginx => nginx proxy}/nginx.conf (100%) rename {intern-nginx => nginx proxy}/port_map.js (100%) rename {intern-nginx => nginx proxy}/reverse_proxy.conf (100%) diff --git a/container creation/automaticLXCDeployment.sh b/container creation/automaticLXCDeployment.sh new file mode 100644 index 00000000..20c652a0 --- /dev/null +++ b/container creation/automaticLXCDeployment.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Automation Script for attempting to automatically deploy projects and services on a container +# Last Modifided by Maxwell Klema on July 9th, 2025 +# ----------------------------------------------------- + +echo "🚀 Attempting Automatic Deployment" + +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") + +# Clone github repository from correct branch + +pct enter $NEXT_ID < .env +EOF + +# Install correct runtime and project dependencies +if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then +pct enter $NEXT_ID < Date: Thu, 10 Jul 2025 18:32:44 +0000 Subject: [PATCH 07/25] alpha version of automatic deployment working --- container creation/automaticLXCDeployment.sh | 13 +++-- container creation/create-container.sh | 50 ++++++++++++++++++-- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/container creation/automaticLXCDeployment.sh b/container creation/automaticLXCDeployment.sh index 20c652a0..75786f0f 100644 --- a/container creation/automaticLXCDeployment.sh +++ b/container creation/automaticLXCDeployment.sh @@ -1,6 +1,6 @@ #!/bin/bash # Automation Script for attempting to automatically deploy projects and services on a container -# Last Modifided by Maxwell Klema on July 9th, 2025 +# Last Modifided by Maxwell Klema on July 10th, 2025 # ----------------------------------------------------- echo "🚀 Attempting Automatic Deployment" @@ -25,6 +25,7 @@ if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then pct enter $NEXT_ID < /dev/null 2>&1 ssh root@10.15.0.5 "pct start $NEXT_ID" + if [ "${DEPLOY_ON_START^^}" == "Y" ]; then + startProject + fi fi # Echo Container Details -# Define friendly, high-contrast colors BOLD='\033[1m' BLUE='\033[34m' MAGENTA='\033[35m' @@ -107,4 +149,4 @@ echo -e "📦 ${BLUE}Container ID :${RESET} $NEXT_ID" echo -e "🌐 ${MAGENTA}Internal IP :${RESET} $CONTAINER_IP" echo -e "🔗 ${GREEN}Domain Name :${RESET} https://$CONTAINER_NAME.opensource.mieweb.org" echo -e "🛠️ ${BLUE}SSH Access :${RESET} ssh -p $SSH_PORT root@$CONTAINER_NAME.opensource.mieweb.org" -echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" \ No newline at end of file +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" From d89e67e5c5d53ee5851643154ff2c7ca44627bf9 Mon Sep 17 00:00:00 2001 From: maxklema Date: Fri, 11 Jul 2025 00:12:17 +0000 Subject: [PATCH 08/25] automatic deployment for python projects --- container creation/automaticLXCDeployment.sh | 49 +++++++++--- container creation/create-container.sh | 82 ++++++++++++++------ 2 files changed, 95 insertions(+), 36 deletions(-) diff --git a/container creation/automaticLXCDeployment.sh b/container creation/automaticLXCDeployment.sh index 75786f0f..2d313672 100644 --- a/container creation/automaticLXCDeployment.sh +++ b/container creation/automaticLXCDeployment.sh @@ -1,6 +1,6 @@ #!/bin/bash # Automation Script for attempting to automatically deploy projects and services on a container -# Last Modifided by Maxwell Klema on July 10th, 2025 +# Last Modifided by Maxwell Klema on July 9th, 2025 # ----------------------------------------------------- echo "🚀 Attempting Automatic Deployment" @@ -22,30 +22,55 @@ EOF # Install correct runtime and project dependencies if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then -pct enter $NEXT_ID < ~/.ssh/authorized_keys"< /var/lib/vz/snippets/container-public-keys/$PUB_FILE @@ -89,16 +94,24 @@ pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" if [ "${DEPLOY_ON_START^^}" == "Y" ]; then source /var/lib/vz/snippets/deployOnStart.sh + + #cleanup + if [ -f "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE" ]; then + rm -rf "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE" + fi + if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then + rm -rf "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" + fi fi # Run Contianer Provision Script to add container to port_map.json if [ -f "/var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE" ]; then echo "CONTAINS PROTOCOL FILE" - /var/lib/vz/snippets/register-container-test.sh $NEXT_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE + /var/lib/vz/snippets/register-container.sh $NEXT_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE rm -rf /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE else - /var/lib/vz/snippets/register-container-test.sh $NEXT_ID $HTTP_PORT + /var/lib/vz/snippets/register-container.sh $NEXT_ID $HTTP_PORT fi SSH_PORT=$(iptables -t nat -S PREROUTING | grep "to-destination $CONTAINER_IP:22" | awk -F'--dport ' '{print $2}' | awk '{print $1}' | head -n 1 || true) @@ -106,28 +119,53 @@ SSH_PORT=$(iptables -t nat -S PREROUTING | grep "to-destination $CONTAINER_IP:22 # Migrate to pve2 if Container ID is even startProject() { -if [ "$BUILD_COMMAND" == "" ]; then -ssh root@10.15.0.5 " -pct enter $NEXT_ID < /dev/null 2>&1 + fi +elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then + if [ "$BUILD_COMMAND" == "" ]; then +ssh root@10.15.0.5 " +pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && \ +pct enter $NEXT_ID < /dev/null 2>&1 + fi fi } if (( $NEXT_ID % 2 == 0 )); then - pct stop $NEXT_ID + pct stop $NEXT_ID > /dev/null 2>&1 pct migrate $NEXT_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 - ssh root@10.15.0.5 "pct start $NEXT_ID" + ssh root@10.15.0.5 "pct start $NEXT_ID" > /dev/null 2>&1 if [ "${DEPLOY_ON_START^^}" == "Y" ]; then startProject fi @@ -141,10 +179,6 @@ MAGENTA='\033[35m' GREEN='\033[32m' RESET='\033[0m' -if (( $NEXT_ID % 2 == 0 )); then - echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -fi - echo -e "📦 ${BLUE}Container ID :${RESET} $NEXT_ID" echo -e "🌐 ${MAGENTA}Internal IP :${RESET} $CONTAINER_IP" echo -e "🔗 ${GREEN}Domain Name :${RESET} https://$CONTAINER_NAME.opensource.mieweb.org" From 12ecc106003cea02d98b444c23bca938efca80c4 Mon Sep 17 00:00:00 2001 From: maxklema Date: Fri, 11 Jul 2025 17:30:30 +0000 Subject: [PATCH 09/25] python processes running in background via tmux --- container creation/automaticLXCDeployment.sh | 9 ++---- container creation/create-container.sh | 30 +++++++++----------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/container creation/automaticLXCDeployment.sh b/container creation/automaticLXCDeployment.sh index 2d313672..7817c282 100644 --- a/container creation/automaticLXCDeployment.sh +++ b/container creation/automaticLXCDeployment.sh @@ -1,6 +1,6 @@ #!/bin/bash # Automation Script for attempting to automatically deploy projects and services on a container -# Last Modifided by Maxwell Klema on July 9th, 2025 +# Last Modifided by Maxwell Klema on July 11th, 2025 # ----------------------------------------------------- echo "🚀 Attempting Automatic Deployment" @@ -30,7 +30,7 @@ cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && sudo $INSTALL_COMMAND EOF elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then pct enter $NEXT_ID < /dev/null 2>&1 fi elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then - if [ "$BUILD_COMMAND" == "" ]; then + if [ "$BUILD_COMMAND" == "" ]; then +ssh -T root@10.15.0.5 " +pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && \ +pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate && $START_COMMAND'\" /dev/null && \ +pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 +" > /dev/null 2>&1 + else ssh root@10.15.0.5 " pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && \ -pct enter $NEXT_ID < /dev/null 2>&1 - fi +" > /dev/null 2>&1 + fi fi + } if (( $NEXT_ID % 2 == 0 )); then From 9d742a8821fc91ba65d9bd8a8b98fe9b904cdfe8 Mon Sep 17 00:00:00 2001 From: maxklema Date: Fri, 11 Jul 2025 23:56:23 +0000 Subject: [PATCH 10/25] working container updation script --- container creation/create-container.sh | 41 ++-- container creation/get-deployment-details.sh | 46 ++--- container updates/update-container.sh | 204 +++++++++++++++++++ jump server/extract-fingerprint.sh | 22 +- 4 files changed, 262 insertions(+), 51 deletions(-) create mode 100644 container updates/update-container.sh diff --git a/container creation/create-container.sh b/container creation/create-container.sh index 10ddbd74..4a7c932b 100644 --- a/container creation/create-container.sh +++ b/container creation/create-container.sh @@ -122,37 +122,30 @@ startProject() { if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then if [ "$BUILD_COMMAND" == "" ]; then ssh root@10.15.0.5 " - pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && - pct enter $NEXT_ID < /dev/null 2>&1 else ssh root@10.15.0.5 " - pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && - pct enter $NEXT_ID < /dev/null 2>&1 fi elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then if [ "$BUILD_COMMAND" == "" ]; then -ssh -T root@10.15.0.5 " -pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && \ -pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate && $START_COMMAND'\" /dev/null && \ -pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 -" > /dev/null 2>&1 + ssh -T root@10.15.0.5 " + pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && \ + pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate && $START_COMMAND'\" /dev/null && \ + pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 + " > /dev/null 2>&1 else -ssh root@10.15.0.5 " -pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && \ -pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate $BUILD_COMMAND && $START_COMMAND'\" /dev/null && \ -pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 -" > /dev/null 2>&1 + ssh root@10.15.0.5 " + pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && \ + pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate $BUILD_COMMAND && $START_COMMAND'\" /dev/null && \ + pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 + " > /dev/null 2>&1 fi fi diff --git a/container creation/get-deployment-details.sh b/container creation/get-deployment-details.sh index 81682486..82809ec7 100755 --- a/container creation/get-deployment-details.sh +++ b/container creation/get-deployment-details.sh @@ -8,9 +8,9 @@ RESET="\033[0m" BOLD="\033[1m" MAGENTA='\033[35m' -echo -e "${BOLD}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" echo -e "${BOLD}${MAGENTA}🌐 Let's Get Your Project Automatically Deployed ${RESET}" -echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n${RESET}" +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" # Get and validate project repository ======== @@ -18,10 +18,18 @@ if [ -z "$PROJECT_REPOSITORY" ]; then read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY fi +CheckRepository() { + PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY#*github.com/} + PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY_SHORTENED%.git} + REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED) +} + +CheckRepository -while ! git ls-remote --heads "$PROJECT_REPOSITORY" > /dev/null 2>&1 ; do +while [ "$REPOSITORY_EXISTS" != "200" ]; do echo "⚠️ The repository link you provided, \"$PROJECT_REPOSITORY\" was not valid." read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY + CheckRepository done # Get Repository Branch ======== @@ -34,9 +42,12 @@ if [ "$PROJECT_BRANCH" == "" ]; then PROJECT_BRANCH="main" fi -while ! git ls-remote --heads "$PROJECT_REPOSITORY" | grep -q "refs/heads/$PROJECT_BRANCH"; do +REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED/branches/$PROJECT_BRANCH) + +while [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; do echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH + REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED/branches/$PROJECT_BRANCH) done # Get Project Root Directory ======== @@ -49,14 +60,19 @@ if [ "$PROJECT_ROOT" == "" ]; then PROJECT_ROOT="/" fi -VALID_PROJECT_ROOT=$(node js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") +VALID_PROJECT_ROOT=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") while [ "$VALID_PROJECT_ROOT" == "false" ]; do echo "⚠️ The root directory you provided, \"$PROJECT_ROOT\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT - VALID_PROJECT_ROOT=$(node js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") + VALID_PROJECT_ROOT=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") done +# Remove forward slash +if [[ "$PROJECT_ROOT" == "/*" ]]; then + PROJECT_ROOT="${PROJECT_ROOT:1}" +fi + # Get Environment Variables ======== gatherEnvVars(){ @@ -113,28 +129,12 @@ if [ -z "$INSTALL_COMMAND" ]; then read -p "📦 Enter the install command (e.g., 'npm install') → " INSTALL_COMMAND fi -while [ "$INSTALL_COMMAND" == "" ]; do - echo "⚠️ The install command cannot be blank. Please try again." - read -p "📦 Enter the install command (e.g., 'npm install') → " INSTALL_COMMAND -done - # Get Build Command ======== if [ -z "$BUILD_COMMAND" ]; then read -p "🏗️ Enter the build command (leave blank if no build command) → " BUILD_COMMAND fi -# Get Build Directory ======== - -if [ -z "$BUILD_DIRECTORY" ]; then - read -p "📂 Enter the build directory (e.g., 'dist'; leave blank if not applicable) → " BUILD_DIRECTORY -fi - -while [ "$BUILD_COMMAND" == "" ] && [ "$BUILD_DIRECTORY" != "" ]; do - echo "⚠️ You did not enter a build command. The build directory should be empty, too. Please try again." - read -p "📂 Enter the build directory (e.g., 'dist', 'build', leave blank if not applicable) → " BUILD_DIRECTORY -done - # Get Start Command ======== if [ -z "$START_COMMAND" ]; then @@ -159,7 +159,7 @@ done # Get Services ======== -SERVICE_MAP="services/service_map.json" +SERVICE_MAP="/root/bin/services/service_map.json" APPENDED_SERVICES=() # Helper function to check if a user has added the same service twice diff --git a/container updates/update-container.sh b/container updates/update-container.sh new file mode 100644 index 00000000..4b6eb4c8 --- /dev/null +++ b/container updates/update-container.sh @@ -0,0 +1,204 @@ +#!/bin/bash +# Script to automatically fetch new contents from a branch, push them to container, and restart intern +# Last Modified on July 11th, 2025 by Maxwell Klema +# ---------------------------------------- + +# INSTALL_COMMAND BUILD_COMMAND START_COMMAND RUNTIME_LANGUAGE + +RESET="\033[0m" +BOLD="\033[1m" +MAGENTA='\033[35m' + +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +echo -e "${BOLD}${MAGENTA}🔄 Update Container Contents ${RESET}" +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + +# Authenticate User (Only Valid Users can Create Containers) + +if [ -z "$PROXMOX_USERNAME" ]; then + read -p "Enter Proxmox Username → " PROXMOX_USERNAME +fi + +if [ -z "$PROXMOX_PASSWORD" ]; then + read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD + echo "" +fi + +USER_AUTHENTICATED=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateUser \"$PROXMOX_USERNAME\" \"$PROXMOX_PASSWORD\"") +RETRIES=3 + +while [ $USER_AUTHENTICATED == 'false' ]; do + if [ $RETRIES -gt 0 ]; then + echo "❌ Authentication Failed. Try Again" + read -p "Enter Proxmox Username → " PROXMOX_USERNAME + read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD + echo "" + + USER_AUTHENTICATED=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateUser \"$PROXMOX_USERNAME\" \"$PROXMOX_PASSWORD\"") + RETRIES=$(($RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 2 + fi +done + +echo "🎉 Your proxmox account, $PROXMOX_USERNAME@pve, has been authenticated" + +# Get CTID from Container Name + +if [ -z "$CONTAINER_NAME" ]; then + read -p "Enter Container Name → " CONTAINER_NAME +fi + +CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; }| awk -v name="$CONTAINER_NAME" '$3 == name {print $1}') + +if [ -z "$CONTAINER_ID" ]; then + echo "❌ Container with name $CONTAINER_NAME does not exist on your account." + exit 1 +fi + +if (( $CONTAINER_ID % 2 == 0 )); then + CONTAINER_OWNERSHIP=$(ssh root@10.15.0.5 "pct config \"$CONTAINER_ID\" | grep "tags" | grep \"$PROXMOX_USERNAME\"") +else + CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep "$PROXMOX_USERNAME") +fi + +# echo "$CONTAINER_OWNERSHIP" + +if [ -z "$CONTAINER_OWNERSHIP" ]; then + echo "❌ You do not own the container with name $CONTAINER_NAME." + exit 1 +fi + +# Get Project Details + +if [ -z "$PROJECT_REPOSITORY" ]; then + read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY +fi + +CheckRepository() { + PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY#*github.com/} + PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY_SHORTENED%.git} + REPOSITORY_EXISTS=$(curl -H "Authorization: token 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" -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED) +} + +CheckRepository + +while [ "$REPOSITORY_EXISTS" != "200" ]; do + echo "⚠️ The repository link you provided, \"$PROJECT_REPOSITORY\" was not valid." + read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY + CheckRepository +done + +echo "✅ The repository link you provided, \"$PROJECT_REPOSITORY\", was valid." + +# Get Project Branch + +if [ -z "$PROJECT_BRANCH" ]; then + read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH +fi + +if [ "$PROJECT_BRANCH" == "" ]; then + PROJECT_BRANCH="main" +fi + +REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED/branches/$PROJECT_BRANCH) + +while [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; do + echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." + read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH + REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED/branches/$PROJECT_BRANCH) +done + + +# Get Project Root Directroy + +if [ -z "$PROJECT_ROOT" ]; then + read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT +fi + +if [ "$PROJECT_ROOT" == "" ]; then + PROJECT_ROOT="/" +fi + +VALID_PROJECT_ROOT=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateRepo \"$PROJECT_REPOSITORY\" \"$PROJECT_BRANCH\" \"$PROJECT_ROOT\"") + +while [ "$VALID_PROJECT_ROOT" == "false" ]; do + echo "⚠️ The root directory you provided, \"$PROJECT_ROOT\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." + read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT + VALID_PROJECT_ROOT=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateRepo \"$PROJECT_REPOSITORY\" \"$PROJECT_BRANCH\" \"$PROJECT_ROOT\"") +done + +# Get Install Command ======== + +if [ -z "$INSTALL_COMMAND" ]; then + read -p "📦 Enter the install command (e.g., 'npm install') → " INSTALL_COMMAND +fi + +# Get Build Command ======== + +if [ -z "$BUILD_COMMAND" ]; then + read -p "🏗️ Enter the build command (leave blank if no build command) → " BUILD_COMMAND +fi + +# Get Start Command ======== + +if [ -z "$START_COMMAND" ]; then + read -p "🚦 Enter the start command (e.g., 'npm start', 'python app.py') → " START_COMMAND +fi + +while [ "$START_COMMAND" == "" ]; do + echo "⚠️ The start command cannot be blank. Please try again." + read -p "🚦 Enter the start command (e.g., 'npm start') → " START_COMMAND +done + +# Get Runtime Language ======== + +if [ -z "$RUNTIME_LANGUAGE" ]; then + read -p "🖥️ Enter the underlying runtime environment (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE +fi + +while [ "${RUNTIME_LANGUAGE^^}" != "NODEJS" ] && [ "${RUNTIME_LANGUAGE^^}" != "PYTHON" ]; do + echo "⚠️ Sorry, that runtime environment is not yet supported. Only \"nodejs\" and \"python\" are currently supported." + read -p "🖥️ Enter the underlying runtime environment (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE +done + +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") + +# Update Container with New Contents from repository + +if (( "$CONTAINER_ID" % 2 == 0 )); then + if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then + ssh root@10.15.0.5 " + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && + pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' + pct exec $CONTAINER_ID -- bash -c '$INSTALL_COMMAND_COMMAND' && '$BUILD_COMMAND' + pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && pm2 start bash -- -c \"$START_COMMAND\"' + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 + " + elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then + ssh root@10.15.0.5 " + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && + pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' + pct exec $CONTAINER_ID -- bash -c '$INSTALL_COMMAND' && '$BUILD_COMMAND' + pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate && $START_COMMAND'\" /dev/null + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 + " + fi +else + if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && + pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' + pct exec $CONTAINER_ID -- bash -c '$INSTALL_COMMAND_COMMAND' && '$BUILD_COMMAND' + pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && pm2 start bash -- -c \"$START_COMMAND\"' + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 + elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && + pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' + pct exec $CONTAINER_ID -- bash -c '$INSTALL_COMMAND' && '$BUILD_COMMAND' + pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate && $START_COMMAND'\" /dev/null + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 + fi +fi + +echo "✅ Container $CONTAINER_ID has been updated with new contents from branch \"$PROJECT_BRANCH\" on repository \"$PROJECT_REPOSITORY\"." \ No newline at end of file diff --git a/jump server/extract-fingerprint.sh b/jump server/extract-fingerprint.sh index 42c5e0df..dac307b8 100644 --- a/jump server/extract-fingerprint.sh +++ b/jump server/extract-fingerprint.sh @@ -22,11 +22,25 @@ if [ "$diff" -ge 0 ] && [ "$diff" -le 2 ]; then KEY_FINGERPRINT=$(echo $RECENT_LOG | grep -o 'SHA256[^ ]*') fi -export SSH_KEY_FP="$KEY_FINGERPRINT" +# Export environment variables +export PUBLIC_KEY="$PUBLIC_KEY" export PROXMOX_USERNAME="$PROXMOX_USERNAME" export PROXMOX_PASSWORD="$PROXMOX_PASSWORD" export CONTAINER_NAME="$CONTAINER_NAME" export CONTAINER_PASSWORD="$CONTAINER_PASSWORD" -export PUBLIC_KEY="$PUBLIC_KEY" - -ssh -o "SendEnv=SSH_KEY_FP PROXMOX_USERNAME PROXMOX_PASSWORD CONTAINER_NAME CONTAINER_PASSWORD PUBLIC_KEY" -A create-container@10.15.234.122 \ No newline at end of file +export HTTP_PORT="$HTTP_PORT" +export PROJECT_REPOSITORY="$PROJECT_REPOSITORY" +export PROJECT_BRANCH="$PROJECT_BRANCH" +export PROJECT_ROOT="$PROJECT_ROOT" +export REQUIRE_ENV_VARS="$REQUIRE_ENV_VARS" +export CONTAINER_ENV_VARS="$CONTAINER_ENV_VARS" +export INSTALL_COMMAND="$INSTALL_COMMAND" +export BUILD_COMMAND="$BUILD_COMMAND" +export START_COMMAND="$START_COMMAND" +export RUNTIME_LANGUAGE="$RUNTIME_LANGUAGE" +export SERVICES="$SERVICES" +export REQUIRE_SERVICES="$REQUIRE_SERVICES" +export CUSTOM_SERVICES="$CUSTOM_SERVICES" + +# SSH with all SendEnv flags +ssh -o "SendEnv=PUBLIC_KEY PROXMOX_USERNAME PROXMOX_PASSWORD CONTAINER_NAME CONTAINER_PASSWORD HTTP_PORT PROJECT_REPOSITORY PROJECT_BRANCH PROJECT_ROOT REQUIRE_ENV_VARS CONTAINER_ENV_VARS INSTALL_COMMAND BUILD_COMMAND START_COMMAND RUNTIME_LANGUAGE SERVICES REQUIRE_SERVICES CUSTOM_SERVICES" -A create-container@10.15.234.122 From 0ab5b1d5ab8f7313be7b2296db7529e5a1923c38 Mon Sep 17 00:00:00 2001 From: maxklema Date: Fri, 11 Jul 2025 23:58:44 +0000 Subject: [PATCH 11/25] script to automatically update container content on push --- container updates/update-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container updates/update-container.sh b/container updates/update-container.sh index 4b6eb4c8..b464725c 100644 --- a/container updates/update-container.sh +++ b/container updates/update-container.sh @@ -79,7 +79,7 @@ fi CheckRepository() { PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY#*github.com/} PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY_SHORTENED%.git} - REPOSITORY_EXISTS=$(curl -H "Authorization: token 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" -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED) + REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED) } CheckRepository From 0d7b3b69d45fa6da544afc8dda1dee233564fa0e Mon Sep 17 00:00:00 2001 From: maxklema Date: Mon, 14 Jul 2025 20:37:49 +0000 Subject: [PATCH 12/25] Container Maintenance Scripts, Re-factoring, bug fixes, etc. --- container creation/automaticLXCDeployment.sh | 24 +- container creation/create-container.sh | 31 +- container creation/get-deployment-details.sh | 14 +- .../get-lxc-container-details.sh | 445 ++++++++++-------- .../check-container-exists.sh | 17 + container maintenance/delete-container.sh | 39 ++ .../helper-scripts/PVE_user_authentication.sh | 35 ++ .../verify_container_ownership.sh | 26 + .../update-container.sh | 80 +--- 9 files changed, 399 insertions(+), 312 deletions(-) create mode 100644 container maintenance/check-container-exists.sh create mode 100644 container maintenance/delete-container.sh create mode 100644 container maintenance/helper-scripts/PVE_user_authentication.sh create mode 100644 container maintenance/helper-scripts/verify_container_ownership.sh rename {container updates => container maintenance}/update-container.sh (71%) diff --git a/container creation/automaticLXCDeployment.sh b/container creation/automaticLXCDeployment.sh index 7817c282..582f9b7f 100644 --- a/container creation/automaticLXCDeployment.sh +++ b/container creation/automaticLXCDeployment.sh @@ -10,26 +10,26 @@ REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") # Clone github repository from correct branch pct enter $NEXT_ID < /dev/null 2>&1 EOF # Copy over ENV variables ENV_VARS=$(cat /var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE) pct enter $NEXT_ID < .env +cd /root/$REPO_BASE_NAME/ && cd $PROJECT_ROOT && echo "$ENV_VARS" > .env EOF # Install correct runtime and project dependencies if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then - pct enter $NEXT_ID < /dev/null sudo apt-get update -y && \ sudo apt install -y nodejs npm && \ npm install -g pm2 && \ cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && sudo $INSTALL_COMMAND EOF elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then - pct enter $NEXT_ID < /dev/null sudo apt install -y python3-venv python3-pip tmux && \ cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && \ python3 -m venv venv && source venv/bin/activate && \ @@ -41,21 +41,21 @@ fi # Iterate over each service installation command if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then while read line; do - pct exec $NEXT_ID -- bash -c "$line" + pct exec $NEXT_ID -- bash -c "$line" > /dev/null done < "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" fi # Build Project (If Needed) and Start it if (( $NEXT_ID % 2 == 1)); then -pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 #temporarily bump up container resources for computation hungry processes (e.g. meteor)\ +pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 > /dev/null #temporarily bump up container resources for computation hungry processes (e.g. meteor)\ if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then if [ "$BUILD_COMMAND" == "" ]; then - pct exec $NEXT_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && pm2 start bash -- -c '$START_COMMAND'" + pct exec $NEXT_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && pm2 start bash -- -c '$START_COMMAND'" > /dev/null else - pct enter $NEXT_ID < /dev/null cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && \ export PATH=\$PATH:/usr/local/bin && \ $BUILD_COMMAND && pm2 start bash -- -c '$START_COMMAND' @@ -64,10 +64,10 @@ EOF elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then if [ "$BUILD_COMMAND" == "" ]; then - pct exec $NEXT_ID -- "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $START_COMMAND" + pct exec $NEXT_ID -- "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $START_COMMAND" > /dev/null else - pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate $BUILD_COMMAND && $START_COMMAND'\" /dev/null && \ + pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate $BUILD_COMMAND && $START_COMMAND'\" /dev/null fi fi -pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 +pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 > /dev/null fi diff --git a/container creation/create-container.sh b/container creation/create-container.sh index 4a7c932b..a28485a7 100644 --- a/container creation/create-container.sh +++ b/container creation/create-container.sh @@ -59,17 +59,17 @@ function cleanup() echo "⏳ Cloning Container..." pct clone 114 $NEXT_ID \ --hostname $CONTAINER_NAME \ - --full true \ + --full true > /dev/null 2>&1 # Set Container Options echo "⏳ Setting Container Properties.." pct set $NEXT_ID \ --tags "$PROXMOX_USERNAME" \ - --onboot 1 \ + --onboot 1 > /dev/null 2>&1 -pct start $NEXT_ID -pveum aclmod /vms/$NEXT_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser +pct start $NEXT_ID > /dev/null 2>&1 +pveum aclmod /vms/$NEXT_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser > /dev/null 2>&1 #pct delete $NEXT_ID # Get the Container IP Address and install some packages @@ -78,40 +78,39 @@ echo "⏳ Waiting for DHCP to allocate IP address to container..." sleep 10 CONTAINER_IP=$(pct exec $NEXT_ID -- hostname -I | awk '{print $1}') -pct exec $NEXT_ID -- apt-get upgrade -pct exec $NEXT_ID -- apt install -y sudo git curl vim +pct exec $NEXT_ID -- apt-get upgrade > /dev/null 2>&1 +pct exec $NEXT_ID -- apt install -y sudo git curl vim > /dev/null 2>&1 if [ -f "/var/lib/vz/snippets/container-public-keys/$PUB_FILE" ]; then - pct exec $NEXT_ID -- touch ~/.ssh/authorized_keys - pct exec $NEXT_ID -- bash -c "cat > ~/.ssh/authorized_keys"< /var/lib/vz/snippets/container-public-keys/$PUB_FILE - rm -rf /var/lib/vz/snippets/container-public-keys/$PUB_FILE + pct exec $NEXT_ID -- touch ~/.ssh/authorized_keys > /dev/null 2>&1 + pct exec $NEXT_ID -- bash -c "cat > ~/.ssh/authorized_keys"< /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 + rm -rf /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 fi # Set password inside the container -pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" +pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 # Attempt to Automatically Deploy Project Inside Container if [ "${DEPLOY_ON_START^^}" == "Y" ]; then - source /var/lib/vz/snippets/deployOnStart.sh + source /var/lib/vz/snippets/helper-scripts/deployOnStart.sh #cleanup if [ -f "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE" ]; then - rm -rf "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE" + rm -rf "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE" > /dev/null 2>&1 fi if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then - rm -rf "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" + rm -rf "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" > /dev/null 2>&1 fi fi # Run Contianer Provision Script to add container to port_map.json if [ -f "/var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE" ]; then - echo "CONTAINS PROTOCOL FILE" /var/lib/vz/snippets/register-container.sh $NEXT_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE - rm -rf /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE + rm -rf /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE > /dev/null 2>&1 else - /var/lib/vz/snippets/register-container.sh $NEXT_ID $HTTP_PORT + /var/lib/vz/snippets/register-container.sh $NEXT_ID $HTTP_PORT fi SSH_PORT=$(iptables -t nat -S PREROUTING | grep "to-destination $CONTAINER_IP:22" | awk -F'--dport ' '{print $2}' | awk '{print $1}' | head -n 1 || true) diff --git a/container creation/get-deployment-details.sh b/container creation/get-deployment-details.sh index 82809ec7..b0975c52 100755 --- a/container creation/get-deployment-details.sh +++ b/container creation/get-deployment-details.sh @@ -21,7 +21,7 @@ fi CheckRepository() { PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY#*github.com/} PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY_SHORTENED%.git} - REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED) + REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$RROJECT_REPOSITORY) } CheckRepository @@ -42,12 +42,14 @@ if [ "$PROJECT_BRANCH" == "" ]; then PROJECT_BRANCH="main" fi -REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED/branches/$PROJECT_BRANCH) - +REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY/tree/$PROJECT_BRANCH) while [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; do echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH - REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED/branches/$PROJECT_BRANCH) + if [ "PROJECT_BRANCH" == "" ]; then + PROJECT_BRANCH="main" + fi + REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY_SHORTENED/tree/$PROJECT_BRANCH) done # Get Project Root Directory ======== @@ -56,10 +58,6 @@ if [ -z "$PROJECT_ROOT" ]; then read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT fi -if [ "$PROJECT_ROOT" == "" ]; then - PROJECT_ROOT="/" -fi - VALID_PROJECT_ROOT=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") while [ "$VALID_PROJECT_ROOT" == "false" ]; do diff --git a/container creation/get-lxc-container-details.sh b/container creation/get-lxc-container-details.sh index 0f3def64..b0975c52 100644 --- a/container creation/get-lxc-container-details.sh +++ b/container creation/get-lxc-container-details.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Main Container Creation Script -# Modified June 23rd, 2025 by Maxwell Klema +# Helper script to gather project details for automatic deployment +# Modified July 9th, 2025 by Maxwell Klema # ------------------------------------------ # Define color variables (works on both light and dark backgrounds) @@ -9,246 +9,277 @@ BOLD="\033[1m" MAGENTA='\033[35m' echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -echo -e "${BOLD}${MAGENTA}📦 MIE Container Creation Script ${RESET}" +echo -e "${BOLD}${MAGENTA}🌐 Let's Get Your Project Automatically Deployed ${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -# Authenticate User (Only Valid Users can Create Containers) +# Get and validate project repository ======== -if [ -z "$PROXMOX_USERNAME" ]; then - read -p "Enter Proxmox Username → " PROXMOX_USERNAME +if [ -z "$PROJECT_REPOSITORY" ]; then + read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY fi -if [ -z "$PROXMOX_PASSWORD" ]; then - read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD - echo "" -fi +CheckRepository() { + PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY#*github.com/} + PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY_SHORTENED%.git} + REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$RROJECT_REPOSITORY) +} + +CheckRepository -USER_AUTHENTICATED=$(node /root/bin/js/authenticateUserRunner.js authenticateUser "$PROXMOX_USERNAME" "$PROXMOX_PASSWORD") -RETRIES=3 - -while [ $USER_AUTHENTICATED == 'false' ]; do - if [ $RETRIES -gt 0 ]; then - echo "❌ Authentication Failed. Try Again" - read -p "Enter Proxmox Username → " PROXMOX_USERNAME - read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD - echo "" - - USER_AUTHENTICATED=$(node /root/bin/js/authenticateUserRunner.js authenticateUser "$PROXMOX_USERNAME" "$PROXMOX_PASSWORD") - RETRIES=$(($RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 2 - fi +while [ "$REPOSITORY_EXISTS" != "200" ]; do + echo "⚠️ The repository link you provided, \"$PROJECT_REPOSITORY\" was not valid." + read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY + CheckRepository done -echo "🎉 Your proxmox account, $PROXMOX_USERNAME@pve, has been authenticated" +# Get Repository Branch ======== -# Gather Container Hostname (hostname.opensource.mieweb.org) +if [ -z "$PROJECT_BRANCH" ]; then + read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH +fi -if [ -z "$CONTAINER_NAME" ]; then - read -p "Enter Application Name (One-Word) → " CONTAINER_NAME +if [ "$PROJECT_BRANCH" == "" ]; then + PROJECT_BRANCH="main" fi -CONTAINER_NAME="${CONTAINER_NAME,,}" #convert to lowercase -HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") -HOST_NAME_RETRIES=10 - -while [[ $HOST_NAME_EXISTS == 'true' ]] || [[ ! "$CONTAINER_NAME" =~ ^[A-Za-z0-9-]+$ ]]; do - if [ $HOST_NAME_RETRIES -gt 0 ]; then - echo "Sorry! Either that name has already been registered or your hostname is ill-formatted. Try another name" - read -p "Enter Application Name (One-Word) → " CONTAINER_NAME - HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") - HOST_NAME_RETRIES=$(($HOST_NAME_RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 3 - fi +REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY/tree/$PROJECT_BRANCH) +while [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; do + echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." + read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH + if [ "PROJECT_BRANCH" == "" ]; then + PROJECT_BRANCH="main" + fi + REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY_SHORTENED/tree/$PROJECT_BRANCH) done -echo "✅ $CONTAINER_NAME is available" - -# Gather Container Password -PASSWORD_RETRIES=10 - -if [ -z "$CONTAINER_PASSWORD" ]; then - read -sp "Enter Container Password → " CONTAINER_PASSWORD - echo - read -sp "Confirm Container Password → " CONFIRM_PASSWORD - echo - - while [[ "$CONFIRM_PASSWORD" != "$CONTAINER_PASSWORD" || ${#CONTAINER_PASSWORD} -lt 8 ]]; do - if [ $PASSWORD_RETRIES -gt 0 ]; then - echo "Sorry, try again. Ensure passwords are at least 8 characters." - read -sp "Enter Container Password → " CONTAINER_PASSWORD - echo - read -sp "Confirm Container Password → " CONFIRM_PASSWORD - echo - PASSWORD_RETRIES=$(($PASSWORD_RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 4 - fi - done -else - CONFIRM_PASSWORD="$CONTAINER_PASSWORD" - while [[ "$CONFIRM_PASSWORD" != "$CONTAINER_PASSWORD" || ${#CONTAINER_PASSWORD} -lt 8 ]]; do - if [ $PASSWORD_RETRIES -gt 0 ]; then - echo "Sorry, try again. Ensure passwords are at least 8 characters." - read -sp "Enter Container Password → " CONTAINER_PASSWORD - echo - read -sp "Confirm Container Password → " CONFIRM_PASSWORD - echo - PASSWORD_RETRIES=$(($PASSWORD_RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 4 - fi - done -fi +# Get Project Root Directory ======== -# Attempt to detect public keys - -echo -e "\n🔑 Attempting to Detect SSH Public Key..." - -AUTHORIZED_KEYS="/root/.ssh/authorized_keys" -RANDOM_NUM=$(shuf -i 100000-999999 -n 1) -PUB_FILE="key_$RANDOM_NUM.pub" -TEMP_PUB_FILE="/root/bin/ssh/temp_pubs/$PUB_FILE" # in case two users are running this script at the same time, they do not overwrite each other's temp files -touch "$TEMP_PUB_FILE" -DETECT_PUBLIC_KEY=$(sudo /root/bin/ssh/detectPublicKey.sh "$SSH_KEY_FP" "$TEMP_PUB_FILE") -KEY_RETRIES=10 - -if [ "$DETECT_PUBLIC_KEY" == "Public key found for create-container" ]; then - echo "🔐 Public Key Found!" -else - echo "🔍 Could not detect Public Key" - - if [ -z "$PUBLIC_KEY" ]; then - read -p "Enter Public Key (Allows Easy Access to Container) [OPTIONAL - LEAVE BLANK TO SKIP] → " PUBLIC_KEY - fi - - # Check if key is valid - - while [[ "$PUBLIC_KEY" != "" && $(echo "$PUBLIC_KEY" | ssh-keygen -l -f - 2>&1 | tr -d '\r') == "(stdin) is not a public key file." ]]; do - if [ $KEY_RETRIES -gt 0 ]; then - echo "❌ \"$PUBLIC_KEY\" is not a valid key. Enter either a valid key or leave blank to skip." - read -p "Enter Public Key (Allows Easy Access to Container) [OPTIONAL - LEAVE BLANK TO SKIP] → " PUBLIC_KEY - KEY_RETRIES=$(($KEY_RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 5 - fi - done - - if [ "$PUBLIC_KEY" != "" ]; then - echo "$PUBLIC_KEY" > "$AUTHORIZED_KEYS" && systemctl restart ssh - echo "$PUBLIC_KEY" > "$TEMP_PUB_FILE" - sudo /root/bin/ssh/publicKeyAppendJumpHost.sh "$PUBLIC_KEY" - fi +if [ -z "$PROJECT_ROOT" ]; then + read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT fi -# Get HTTP Port Container Listens On -HTTP_PORT_RETRIES=10 +VALID_PROJECT_ROOT=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") -if [ -z "$HTTP_PORT" ]; then - read -p "Enter HTTP Port for your container to listen on (80-9999) → " HTTP_PORT +while [ "$VALID_PROJECT_ROOT" == "false" ]; do + echo "⚠️ The root directory you provided, \"$PROJECT_ROOT\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." + read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT + VALID_PROJECT_ROOT=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") +done + +# Remove forward slash +if [[ "$PROJECT_ROOT" == "/*" ]]; then + PROJECT_ROOT="${PROJECT_ROOT:1}" fi -while ! [[ "$HTTP_PORT" =~ ^[0-9]+$ ]] || [ "$HTTP_PORT" -lt 80 ] || [ "$HTTP_PORT" -gt 9999 ]; do - if [ $HTTP_PORT_RETRIES -gt 0 ]; then - echo "❌ Invalid HTTP Port. It must be a number between 80 and 9,999." - read -p "Enter HTTP Port for your container to listen on (80-9999) → " HTTP_PORT - HTTP_PORT_RETRIES=$(($HTTP_PORT_RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 6 - fi -done +# Get Environment Variables ======== -echo "✅ HTTP Port is set to $HTTP_PORT" +gatherEnvVars(){ -# Get any other protocols + read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY + read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE -protocol_duplicate() { - PROTOCOL="$1" - shift #remaining params are part of list - LIST="$@" + while [ "$ENV_VAR_KEY" == "" ] || [ "$ENV_VAR_VALUE" == "" ]; do + echo "⚠️ Key or value cannot be empty. Try again." + read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY + read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE + done - for item in $LIST; do - if [[ "$item" == "$PROTOCOL" ]]; then - return 0 # Protocol is a duplicate - fi - done - return 1 # Protocol is not a duplicate + echo "$ENV_VAR_KEY=$ENV_VAR_VALUE" >> $TEMP_ENV_FILE_PATH + + read -p "🔑 Do you want to enter another Environment Variable? (y/n) → " ENTER_ANOTHER_ENV } -read -p "Does your Container require any protocols other than SSH and HTTP? (y/n) → " USE_OTHER_PROTOCOLS -while [ "${USE_OTHER_PROTOCOLS^^}" != "Y" ] && [ "${USE_OTHER_PROTOCOLS^^}" != "N" ] && [ "${USER_OTHER_PROTOCOLS^^}" != "" ]; do - echo "Please answer 'y' for yes or 'n' for no." - read -p "Does your Container require any protocols other than SSH and HTTP? (y/n) → " USE_OTHER_PROTOCOLS +if [ -z "$REQUIRE_ENV_VARS" ]; then + read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS +fi + +while [ "${REQUIRE_ENV_VARS^^}" != "Y" ] && [ "${REQUIRE_ENV_VARS^^}" != "N" ] && [ "${REQUIRE_ENV_VARS^^}" != "" ]; do + echo "⚠️ Invalid option. Please try again." + read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS done -RANDOM_NUM=$(shuf -i 100000-999999 -n 1) -PROTOCOL_BASE_FILE="protocol_list_$RANDOM_NUM.txt" -PROTOCOL_FILE="/root/bin/protocols/$PROTOCOL_BASE_FILE" -touch "$PROTOCOL_FILE" - -if [ "${USE_OTHER_PROTOCOLS^^}" == "Y" ]; then - LIST_PROTOCOLS=() - read -p "Enter the protocol abbreviation (e.g, LDAP for Lightweight Directory Access Protocol). Type \"e\" to exit → " PROTOCOL_NAME - while [ "${PROTOCOL_NAME^^}" != "E" ]; do - FOUND=0 #keep track if protocol was found - while read line; do - PROTOCOL_ABBRV=$(echo "$line" | awk '{print $1}') - protocol_duplicate "$PROTOCOL_ABBRV" "${LIST_PROTOCOLS[@]}" - IS_PROTOCOL_DUPLICATE=$? - if [[ "$PROTOCOL_ABBRV" == "${PROTOCOL_NAME^^}" && "$IS_PROTOCOL_DUPLICATE" -eq 1 ]]; then - LIST_PROTOCOLS+=("$PROTOCOL_ABBRV") - PROTOCOL_UNDRLYING_NAME=$(echo "$line" | awk '{print $3}') - PROTOCOL_DEFAULT_PORT=$(echo "$line" | awk '{print $2}') - echo "$PROTOCOL_ABBRV $PROTOCOL_UNDRLYING_NAME $PROTOCOL_DEFAULT_PORT" >> "$PROTOCOL_FILE" - echo "✅ Protocol ${PROTOCOL_NAME^^} added to container." - FOUND=1 #protocol was found - break - else - echo "❌ Protocol ${PROTOCOL_NAME^^} was already added to your container. Please try again." - FOUND=2 #protocol was a duplicate - break - fi - done < <(cat "/root/bin/protocols/master_protocol_list.txt" | grep "^${PROTOCOL_NAME^^}") - - if [ $FOUND -eq 0 ]; then #if no results found, let user know. - echo "❌ Protocol ${PROTOCOL_NAME^^} not found. Please try again." - fi - - read -p "Enter the protocol abbreviation (e.g, LDAP for Lightweight Directory Access Protocol). Type \"e\" to exit → " PROTOCOL_NAME - done +if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then + + # generate random temp .env file + RANDOM_NUM=$(shuf -i 100000-999999 -n 1) + ENV_FILE="env_$RANDOM_NUM.txt" + TEMP_ENV_FILE_PATH="/root/bin/env/$ENV_FILE" + touch "$TEMP_ENV_FILE_PATH" + + if [ ! -z "$CONTAINER_ENV_VARS" ]; then + if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success + echo "$CONTAINER_ENV_VARS " | jq -r 'to_entries[] | "\(.key)=\(.value)"' > "$TEMP_ENV_FILE_PATH" #k=v pairs + else + echo "⚠️ Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." + exit 10 + fi + else + gatherEnvVars + while [ "${ENTER_ANOTHER_ENV^^}" == "Y" ]; do + gatherEnvVars + done + fi fi -# send public key file & port map file to hypervisor and ssh, Create the Container, run port mapping script +# Get Install Command ======== -if [ -s $TEMP_PUB_FILE ]; then -sftp root@10.15.0.4 <> "$TEMP_SERVICES_FILE_PATH" + echo "sudo systemctl daemon-reload" >> "$TEMP_SERVICES_FILE_PATH" + echo "✅ ${SERVICE^^} added to your container." + APPENDED_SERVICES+=("${SERVICE^^}") + elif [ "${SERVICE^^}" == "C" ]; then + appendCustomService + elif [ "${SERVICE^^}" != "" ]; then + echo "⚠️ Service \"$SERVICE\" does not exist." + [ "$MODE" = "batch" ] && exit 20 + fi +} + +# Helper function to append a new service to a container +appendService() { + if [ ! -z "$SERVICES" ]; then + for SERVICE in $(echo "$SERVICES" | jq -r '.[]'); do + processService "$SERVICE" "batch" + done + else + read -p "➡️ Enter the name of a service to add to your container or type \"C\" to set up a custom service installation (Enter to exit) → " SERVICE + processService "$SERVICE" "single" + fi +} + +appendCustomService() { + # If there is an env variable for custom services, iterate through each command and append it to temporary services file + if [ ! -z "$CUSTOM_SERVICES" ]; then + echo "$CUSTOM_SERVICES" | jq -c -r '.[]' | while read -r CUSTOM_SERVICE; do + echo "$CUSTOM_SERVICE" | jq -c -r '.[]' | while read -r CUSTOM_SERVICE_COMMAND; do + if [ ! -z "$CUSTOM_SERVICE_COMMAND" ]; then + echo "$CUSTOM_SERVICE_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" + else + echo "⚠️ Command cannot be empty." + exit 21; + fi + done + done + echo "✅ Custom Services appended." + else + echo "🛎️ Configuring Custom Service Installation. For each prompt, enter a command that is a part of the installation process for your service on Debian Bookworm. Do not forget to enable and start the service at the end. Once you have entered all of your commands, press enter to continue" + COMMAND_NUM=1 + read -p "➡️ Enter Command $COMMAND_NUM: " CUSTOM_COMMAND + + echo "$CUSTOM_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" + + while [ "${CUSTOM_COMMAND^^}" != "" ]; do + ((COMMAND_NUM++)) + read -p "➡️ Enter Command $COMMAND_NUM: " CUSTOM_COMMAND + echo "$CUSTOM_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" + done + fi +} + +# Helper function to see if a user wants to set up a custom service +setUpService() { + read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION + while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do + echo "⚠️ Invalid option. Please try again." + read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION + done +} + +if [ -z "$REQUIRE_SERVICES" ]; then + read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES +fi + +while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do + echo "⚠️ Invalid option. Please try again." + read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES +done + +if [ "${REQUIRE_SERVICES^^}" == "Y" ]; then + + # Generate random (temporary) file to store install commands for needed services + RANDOM_NUM=$(shuf -i 100000-999999 -n 1) + SERVICES_FILE="services_$RANDOM_NUM.txt" + TEMP_SERVICES_FILE_PATH="/root/bin/services/$SERVICES_FILE" + touch "$TEMP_SERVICES_FILE_PATH" + + appendService + while [ "${SERVICE^^}" != "" ] || [ ! -z "$SERVICES" ]; do + if [ -z "$SERVICES" ]; then + appendService + else + if [ ! -z "$CUSTOM_SERVICES" ]; then # assumes both services and custom services passed as ENV vars + appendCustomService + else # custom services not passed as ENV var, so must prompt the user for their custom services + setUpService + while [ "${SETUP_CUSTOM_SERVICE_INSTALLATION^^}" == "Y" ]; do + appendCustomService + setUpService + done + fi + break + fi + done +fi -unset CONFIRM_PASSWORD -unset CONTAINER_PASSWORD -unset PUBLIC_KEY +echo -e "\n✅ Deployment Process Finished.\n" diff --git a/container maintenance/check-container-exists.sh b/container maintenance/check-container-exists.sh new file mode 100644 index 00000000..669c79df --- /dev/null +++ b/container maintenance/check-container-exists.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Script to check if a container exists +# Last Modified by Maxwell Klema on July 13th, 2025 +# ----------------------------------------------------- + +RESET="\033[0m" +BOLD="\033[1m" +MAGENTA='\033[35m' + +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +echo -e "${BOLD}${MAGENTA}🔎 Check Container Exists ${RESET}" +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + +source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh +source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh + +exit 0 # Container Exists and is owned by the user \ No newline at end of file diff --git a/container maintenance/delete-container.sh b/container maintenance/delete-container.sh new file mode 100644 index 00000000..63cc1b29 --- /dev/null +++ b/container maintenance/delete-container.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Script to delete a container permanently +# Last Modified by Maxwell Klema on July 13th, 2025 +# ----------------------------------------------------- + +RESET="\033[0m" +BOLD="\033[1m" +MAGENTA='\033[35m' + +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +echo -e "${BOLD}${MAGENTA}🗑️ Delete Container ${RESET}" +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + +source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh +source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh + +# Delete Container + +echo "🔄 Deleting container with name \"$CONTAINER_NAME\"..." + +if (( $CONTAINER_ID % 2 == 0 )); then + if ssh root@10.15.0.5 "pct status $CONTAINER_ID" | grep -q "status: running"; then + ssh root@10.15.0.5 "pct stop $CONTAINER_ID && pct destroy $CONTAINER_ID" > /dev/null 2>&1 + else + ssh root@10.15.0.5 "pct destroy $CONTAINER_ID" > /dev/null 2>&1 + fi +else + if pct status "$CONTAINER_ID" | grep -q "status: running"; then + pct stop "$CONTAINER_ID" && pct destroy "$CONTAINER_ID" > /dev/null 2>&1 + else + pct destroy "$CONTAINER_ID" > /dev/null 2>&1 + fi +fi + +echo "🧹 Running Cleanup Tasks..." +source /usr/local/bin/prune_iptables.sh + +echo "✅ Container with name \"$CONTAINER_NAME\" has been permanently deleted." +exit 0 # Container Deleted Successfully \ No newline at end of file diff --git a/container maintenance/helper-scripts/PVE_user_authentication.sh b/container maintenance/helper-scripts/PVE_user_authentication.sh new file mode 100644 index 00000000..c751aadc --- /dev/null +++ b/container maintenance/helper-scripts/PVE_user_authentication.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Script that checks if a user is authenticated in Proxmox PVE Realm @ opensource.mieweb.org +# Last Modified by Maxwell Klema on July 13th, 2025 +# ----------------------------------------------------- + +# Authenticate User (Only Valid Users can Create Containers) + +if [ -z "$PROXMOX_USERNAME" ]; then + read -p "Enter Proxmox Username → " PROXMOX_USERNAME +fi + +if [ -z "$PROXMOX_PASSWORD" ]; then + read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD + echo "" +fi + +USER_AUTHENTICATED=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateUser \"$PROXMOX_USERNAME\" \"$PROXMOX_PASSWORD\"") +RETRIES=3 + +while [ $USER_AUTHENTICATED == 'false' ]; do + if [ $RETRIES -gt 0 ]; then + echo "❌ Authentication Failed. Try Again" + read -p "Enter Proxmox Username → " PROXMOX_USERNAME + read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD + echo "" + + USER_AUTHENTICATED=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateUser \"$PROXMOX_USERNAME\" \"$PROXMOX_PASSWORD\"") + RETRIES=$(($RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 2 + fi +done + +echo "🎉 Your proxmox account, $PROXMOX_USERNAME@pve, has been authenticated" \ No newline at end of file diff --git a/container maintenance/helper-scripts/verify_container_ownership.sh b/container maintenance/helper-scripts/verify_container_ownership.sh new file mode 100644 index 00000000..19ca0541 --- /dev/null +++ b/container maintenance/helper-scripts/verify_container_ownership.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Script to verify container ownership based on name and CTID +# Last Modified by Maxwell Klema on July 13th, 2025 +# ----------------------------------------------------- + +if [ -z "$CONTAINER_NAME" ]; then + read -p "Enter Container Name → " CONTAINER_NAME +fi + +CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$CONTAINER_NAME" '$3 == name {print $1}') + +if [ -z "$CONTAINER_ID" ]; then + echo "❌ Container with name \"$CONTAINER_NAME\" does not exist." + exit 1 +fi + +if (( $CONTAINER_ID % 2 == 0 )); then + CONTAINER_OWNERSHIP=$(ssh root@10.15.0.5 "pct config \"$CONTAINER_ID\" | grep "tags" | grep \"$PROXMOX_USERNAME\"") +else + CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep -x "tags: $PROXMOX_USERNAME") +fi + +if [ -z "$CONTAINER_OWNERSHIP" ]; then + echo "❌ You do not own the container with name \"$CONTAINER_NAME\"." + exit 2 +fi diff --git a/container updates/update-container.sh b/container maintenance/update-container.sh similarity index 71% rename from container updates/update-container.sh rename to container maintenance/update-container.sh index b464725c..4714ac6d 100644 --- a/container updates/update-container.sh +++ b/container maintenance/update-container.sh @@ -3,8 +3,6 @@ # Last Modified on July 11th, 2025 by Maxwell Klema # ---------------------------------------- -# INSTALL_COMMAND BUILD_COMMAND START_COMMAND RUNTIME_LANGUAGE - RESET="\033[0m" BOLD="\033[1m" MAGENTA='\033[35m' @@ -13,62 +11,8 @@ echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━ echo -e "${BOLD}${MAGENTA}🔄 Update Container Contents ${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -# Authenticate User (Only Valid Users can Create Containers) - -if [ -z "$PROXMOX_USERNAME" ]; then - read -p "Enter Proxmox Username → " PROXMOX_USERNAME -fi - -if [ -z "$PROXMOX_PASSWORD" ]; then - read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD - echo "" -fi - -USER_AUTHENTICATED=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateUser \"$PROXMOX_USERNAME\" \"$PROXMOX_PASSWORD\"") -RETRIES=3 - -while [ $USER_AUTHENTICATED == 'false' ]; do - if [ $RETRIES -gt 0 ]; then - echo "❌ Authentication Failed. Try Again" - read -p "Enter Proxmox Username → " PROXMOX_USERNAME - read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD - echo "" - - USER_AUTHENTICATED=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateUser \"$PROXMOX_USERNAME\" \"$PROXMOX_PASSWORD\"") - RETRIES=$(($RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 2 - fi -done - -echo "🎉 Your proxmox account, $PROXMOX_USERNAME@pve, has been authenticated" - -# Get CTID from Container Name - -if [ -z "$CONTAINER_NAME" ]; then - read -p "Enter Container Name → " CONTAINER_NAME -fi - -CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; }| awk -v name="$CONTAINER_NAME" '$3 == name {print $1}') - -if [ -z "$CONTAINER_ID" ]; then - echo "❌ Container with name $CONTAINER_NAME does not exist on your account." - exit 1 -fi - -if (( $CONTAINER_ID % 2 == 0 )); then - CONTAINER_OWNERSHIP=$(ssh root@10.15.0.5 "pct config \"$CONTAINER_ID\" | grep "tags" | grep \"$PROXMOX_USERNAME\"") -else - CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep "$PROXMOX_USERNAME") -fi - -# echo "$CONTAINER_OWNERSHIP" - -if [ -z "$CONTAINER_OWNERSHIP" ]; then - echo "❌ You do not own the container with name $CONTAINER_NAME." - exit 1 -fi +source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh +source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh # Get Project Details @@ -117,10 +61,6 @@ if [ -z "$PROJECT_ROOT" ]; then read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT fi -if [ "$PROJECT_ROOT" == "" ]; then - PROJECT_ROOT="/" -fi - VALID_PROJECT_ROOT=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateRepo \"$PROJECT_REPOSITORY\" \"$PROJECT_BRANCH\" \"$PROJECT_ROOT\"") while [ "$VALID_PROJECT_ROOT" == "false" ]; do @@ -186,19 +126,21 @@ if (( "$CONTAINER_ID" % 2 == 0 )); then " fi else + if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && - pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' - pct exec $CONTAINER_ID -- bash -c '$INSTALL_COMMAND_COMMAND' && '$BUILD_COMMAND' - pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && pm2 start bash -- -c \"$START_COMMAND\"' + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" + pct exec $CONTAINER_ID -- bash -c "$INSTALL_COMMAND_COMMAND' && '$BUILD_COMMAND" + pct exec $CONTAINER_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && pm2 start bash -- -c \"$START_COMMAND\"" pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && - pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' - pct exec $CONTAINER_ID -- bash -c '$INSTALL_COMMAND' && '$BUILD_COMMAND' - pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate && $START_COMMAND'\" /dev/null + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" + pct exec $CONTAINER_ID -- bash -c "$INSTALL_COMMAND' && '$BUILD_COMMAND" + pct exec $CONTAINER_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate && $START_COMMAND'" /dev/null pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 fi fi -echo "✅ Container $CONTAINER_ID has been updated with new contents from branch \"$PROJECT_BRANCH\" on repository \"$PROJECT_REPOSITORY\"." \ No newline at end of file +echo "✅ Container $CONTAINER_ID has been updated with new contents from branch \"$PROJECT_BRANCH\" on repository \"$PROJECT_REPOSITORY\"." +exit 0 \ No newline at end of file From 7cba4db5e5ea3eabaa4e96e932de8c112ace3d0a Mon Sep 17 00:00:00 2001 From: maxklema Date: Tue, 15 Jul 2025 18:57:44 +0000 Subject: [PATCH 13/25] Env vars - multiple component app support --- deployment-scripts/gatherEnvVars.sh | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 deployment-scripts/gatherEnvVars.sh diff --git a/deployment-scripts/gatherEnvVars.sh b/deployment-scripts/gatherEnvVars.sh new file mode 100644 index 00000000..b1a935b2 --- /dev/null +++ b/deployment-scripts/gatherEnvVars.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Helper function to gather environment variables +gatherEnvVars(){ + TEMP_ENV_FILE_PATH="$1" + + read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY + read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE + + while [ "$ENV_VAR_KEY" == "" ] || [ "$ENV_VAR_VALUE" == "" ]; do + echo "⚠️ Key or value cannot be empty. Try again." + read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY + read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE + done + + echo "$ENV_VAR_KEY=$ENV_VAR_VALUE" >> $TEMP_ENV_FILE_PATH + + read -p "🔑 Do you want to enter another Environment Variable? (y/n) → " ENTER_ANOTHER_ENV + + while [ "${ENTER_ANOTHER_ENV^^}" == "Y" ]; do + gatherEnvVars "$TEMP_ENV_FILE_PATH" + done +} + +# Helper functions to gather and validate component directory +gatherComponentDir() { + + COMPONENT_PATH="$1" + if [ -z "$COMPONENT_PATH" ]; then + read -p "Enter the path of a component to store environment variables in, relative to project root directory (To Continue, Press Enter) → " COMPONENT_PATH + fi + # Check that component path is valid + VALID_COMPONENT_PATH=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$COMPONENT_PATH") + while [ "$VALID_COMPONENT_PATH" == "false" ]; do + echo "⚠️ The component path you entered, \"$COMPONENT_PATH\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." + if [ -z "$1" ]; then + read -p "Enter the path of a component to store environment variables in (relative to project root directory) → " COMPONENT_PATH + VALID_COMPONENT_PATH=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$COMPONENT_PATH") + else + exit 9 + fi + done + + if [[ "$COMPONENT_PATH" == /* ]]; then + COMPONENT_PATH="${COMPONENT_PATH:1}" # remove leading slash + fi +} + +if [ -z "$REQUIRE_ENV_VARS" ]; then + read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS +fi + +while [ "${REQUIRE_ENV_VARS^^}" != "Y" ] && [ "${REQUIRE_ENV_VARS^^}" != "N" ] && [ "${REQUIRE_ENV_VARS^^}" != "" ]; do + echo "⚠️ Invalid option. Please try again." + read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS +done + +if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then + + # generate random temp .env folder to store all env files for different components + RANDOM_NUM=$(shuf -i 100000-999999 -n 1) + ENV_FOLDER="env_$RANDOM_NUM" + mkdir -p "/root/bin/env/$ENV_FOLDER" + + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + if [ ! -z "$CONTAINER_ENV_VARS" ]; then # Environment Variables + if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success + for key in $(echo "$CONTAINER_ENV_VARS" | jq -r 'keys[]'); do + gatherComponentDir "$key" + ENV_FILE_NAME=$(echo "$COMPONENT_PATH" | tr '/' '_') + ENV_FILE_NAME="$ENV_FILE_NAME.txt" + ENV_FILE_PATH="/root/bin/env/$ENV_FOLDER/$ENV_FILE_NAME" + touch "$ENV_FILE_PATH" + echo "$CONTAINER_ENV_VARS" | jq -r --arg key "$key" '.[$key] | to_entries[] | "\(.key)=\(.value)"' > "$ENV_FILE_PATH" + done + else + echo "⚠️ Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." + exit 10 + fi + else # No Environment Variables + gatherComponentDir + + while [ "$COMPONENT_PATH" != "" ]; do + ENV_FILE_NAME=$(echo "$COMPONENT_PATH" | tr '/' '_') + ENV_FILE="$ENV_FILE_NAME.txt" + ENV_FILE_PATH="/root/bin/env/$ENV_FOLDER/$ENV_FILE" + touch "$ENV_FILE_PATH" + gatherEnvVars "$ENV_FILE_PATH" + gatherComponentDir + done + fi + else # Single Component + ENV_FILE="env_$RANDOM_NUM.txt" + ENV_FILE_PATH="/root/bin/env/$ENV_FOLDER/$ENV_FILE" + touch "$ENV_FILE_PATH" + if [ ! -z "$CONTAINER_ENV_VARS" ]; then # Environment Variables + if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success + echo "$CONTAINER_ENV_VARS " | jq -r 'to_entries[] | "\(.key)=\(.value)"' > "$ENV_FILE_PATH" #k=v pairs + else + echo "⚠️ Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." + exit 10 + fi + else # No Environment Variables + gatherEnvVars "$ENV_FILE_PATH" + fi + fi +fi \ No newline at end of file From 66e82bb0512bac5358275738821626e20656f37b Mon Sep 17 00:00:00 2001 From: maxklema Date: Tue, 15 Jul 2025 19:50:17 +0000 Subject: [PATCH 14/25] build commands - multi component support --- deployment-scripts/gatherBuildCommands.sh | 32 +++++++++++++++++++++++ deployment-scripts/gatherEnvVars.sh | 31 +++------------------- 2 files changed, 35 insertions(+), 28 deletions(-) create mode 100755 deployment-scripts/gatherBuildCommands.sh diff --git a/deployment-scripts/gatherBuildCommands.sh b/deployment-scripts/gatherBuildCommands.sh new file mode 100755 index 00000000..410804b7 --- /dev/null +++ b/deployment-scripts/gatherBuildCommands.sh @@ -0,0 +1,32 @@ +#!/bin/bash +BUILD_COMMANDS_DICT={} + +if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + if [ ! -z "$BUILD_COMMAND" ]; then # Environment Variable Passed + if echo "$BUILD_COMMAND" | jq -e > /dev/null 2>&1; then # Valid JSON + for key in $(echo "$BUILD_COMMAND" | jq -r 'keys[]'); do + gatherComponentDir "Enter the path of your component to enter the build command" "$key" + done + else + echo "⚠️ Your \"BUILD_COMMANDS\" is not valid JSON. Please re-format and try again." + exit 10 + fi + else # No Environment Variable Passed + gatherComponentDir "Enter the path of your component to enter the build command" + while [ "$COMPONENT_PATH" != "" ]; do + read -p "🏗️ Enter the build command for \"$COMPONENT_PATH\" → " B_COMMAND + + # Append Component:Build_Command k:v pair to map + BUILD_COMMANDS_DICT=$(echo "$BUILD_COMMANDS_DICT" | jq --arg k "$COMPONENT_PATH" --arg v "$B_COMMAND" '. + {($k): $v}') + gatherComponentDir "Enter the path of your component to enter the build command" + done + BUILD_COMMAND=$BUILD_COMMANDS_DICT + fi +else + if [ -z "$BUILD_COMMAND" ]; then + read -p "🏗️ Enter the build command (Press Enter for no Build Command) → " BUILD_COMMAND + fi +fi + +echo "BUILD_COMMAND: $BUILD_COMMAND" + diff --git a/deployment-scripts/gatherEnvVars.sh b/deployment-scripts/gatherEnvVars.sh index b1a935b2..574bdc92 100644 --- a/deployment-scripts/gatherEnvVars.sh +++ b/deployment-scripts/gatherEnvVars.sh @@ -22,30 +22,6 @@ gatherEnvVars(){ done } -# Helper functions to gather and validate component directory -gatherComponentDir() { - - COMPONENT_PATH="$1" - if [ -z "$COMPONENT_PATH" ]; then - read -p "Enter the path of a component to store environment variables in, relative to project root directory (To Continue, Press Enter) → " COMPONENT_PATH - fi - # Check that component path is valid - VALID_COMPONENT_PATH=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$COMPONENT_PATH") - while [ "$VALID_COMPONENT_PATH" == "false" ]; do - echo "⚠️ The component path you entered, \"$COMPONENT_PATH\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." - if [ -z "$1" ]; then - read -p "Enter the path of a component to store environment variables in (relative to project root directory) → " COMPONENT_PATH - VALID_COMPONENT_PATH=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$COMPONENT_PATH") - else - exit 9 - fi - done - - if [[ "$COMPONENT_PATH" == /* ]]; then - COMPONENT_PATH="${COMPONENT_PATH:1}" # remove leading slash - fi -} - if [ -z "$REQUIRE_ENV_VARS" ]; then read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS fi @@ -56,7 +32,6 @@ while [ "${REQUIRE_ENV_VARS^^}" != "Y" ] && [ "${REQUIRE_ENV_VARS^^}" != "N" ] & done if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then - # generate random temp .env folder to store all env files for different components RANDOM_NUM=$(shuf -i 100000-999999 -n 1) ENV_FOLDER="env_$RANDOM_NUM" @@ -66,7 +41,7 @@ if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then if [ ! -z "$CONTAINER_ENV_VARS" ]; then # Environment Variables if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success for key in $(echo "$CONTAINER_ENV_VARS" | jq -r 'keys[]'); do - gatherComponentDir "$key" + gatherComponentDir "Enter the path of your component to store environment variables" "$key" ENV_FILE_NAME=$(echo "$COMPONENT_PATH" | tr '/' '_') ENV_FILE_NAME="$ENV_FILE_NAME.txt" ENV_FILE_PATH="/root/bin/env/$ENV_FOLDER/$ENV_FILE_NAME" @@ -78,7 +53,7 @@ if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then exit 10 fi else # No Environment Variables - gatherComponentDir + gatherComponentDir "Enter the path of your component to store environment variables" while [ "$COMPONENT_PATH" != "" ]; do ENV_FILE_NAME=$(echo "$COMPONENT_PATH" | tr '/' '_') @@ -86,7 +61,7 @@ if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then ENV_FILE_PATH="/root/bin/env/$ENV_FOLDER/$ENV_FILE" touch "$ENV_FILE_PATH" gatherEnvVars "$ENV_FILE_PATH" - gatherComponentDir + gatherComponentDir "Enter the path of your component to store environment variables" done fi else # Single Component From 4a67afc4c1a9d931d188d488e782297b3ccd7093 Mon Sep 17 00:00:00 2001 From: maxklema Date: Tue, 15 Jul 2025 21:53:51 +0000 Subject: [PATCH 15/25] multi compnent runtime and start command support --- deployment-scripts/gatherBuildCommands.sh | 2 +- deployment-scripts/gatherEnvVars.sh | 8 ++-- deployment-scripts/gatherRuntimeLangs.sh | 55 +++++++++++++++++++++++ deployment-scripts/gatherSetupCommands.sh | 51 +++++++++++++++++++++ 4 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 deployment-scripts/gatherRuntimeLangs.sh create mode 100644 deployment-scripts/gatherSetupCommands.sh diff --git a/deployment-scripts/gatherBuildCommands.sh b/deployment-scripts/gatherBuildCommands.sh index 410804b7..12230320 100755 --- a/deployment-scripts/gatherBuildCommands.sh +++ b/deployment-scripts/gatherBuildCommands.sh @@ -8,7 +8,7 @@ if [ "${MULTI_COMPONENT^^}" == "Y" ]; then gatherComponentDir "Enter the path of your component to enter the build command" "$key" done else - echo "⚠️ Your \"BUILD_COMMANDS\" is not valid JSON. Please re-format and try again." + echo "⚠️ Your \"BUILD_COMMAND\" is not valid JSON. Please re-format and try again." exit 10 fi else # No Environment Variable Passed diff --git a/deployment-scripts/gatherEnvVars.sh b/deployment-scripts/gatherEnvVars.sh index 574bdc92..c577edd4 100644 --- a/deployment-scripts/gatherEnvVars.sh +++ b/deployment-scripts/gatherEnvVars.sh @@ -41,27 +41,29 @@ if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then if [ ! -z "$CONTAINER_ENV_VARS" ]; then # Environment Variables if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success for key in $(echo "$CONTAINER_ENV_VARS" | jq -r 'keys[]'); do - gatherComponentDir "Enter the path of your component to store environment variables" "$key" + gatherComponentDir "Enter the path of your component to enter environment variables" "$key" ENV_FILE_NAME=$(echo "$COMPONENT_PATH" | tr '/' '_') ENV_FILE_NAME="$ENV_FILE_NAME.txt" ENV_FILE_PATH="/root/bin/env/$ENV_FOLDER/$ENV_FILE_NAME" touch "$ENV_FILE_PATH" echo "$CONTAINER_ENV_VARS" | jq -r --arg key "$key" '.[$key] | to_entries[] | "\(.key)=\(.value)"' > "$ENV_FILE_PATH" + addComponent "$key" done else echo "⚠️ Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." exit 10 fi else # No Environment Variables - gatherComponentDir "Enter the path of your component to store environment variables" + gatherComponentDir "Enter the path of your component to enter environment variables" while [ "$COMPONENT_PATH" != "" ]; do + addComponent "$key" ENV_FILE_NAME=$(echo "$COMPONENT_PATH" | tr '/' '_') ENV_FILE="$ENV_FILE_NAME.txt" ENV_FILE_PATH="/root/bin/env/$ENV_FOLDER/$ENV_FILE" touch "$ENV_FILE_PATH" gatherEnvVars "$ENV_FILE_PATH" - gatherComponentDir "Enter the path of your component to store environment variables" + gatherComponentDir "Enter the path of your component to enter environment variables" done fi else # Single Component diff --git a/deployment-scripts/gatherRuntimeLangs.sh b/deployment-scripts/gatherRuntimeLangs.sh new file mode 100644 index 00000000..9ed7cae9 --- /dev/null +++ b/deployment-scripts/gatherRuntimeLangs.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +gatherRunTime() { + COMPONENT_PATH="$1" + + if [ -z "${RUNTIME_LANGUAGE}" ] && [ "${MULTI_COMPONENT^^}" == "n" ]; then + read -p "🖥️ Enter the underlying runtime environment for \"$COMPONENT_PATH\" (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE + fi + + while [ "${RUNTIME_LANGUAGE^^}" != "NODEJS" ] && [ "${RUNTIME_LANGUAGE^^}" != "PYTHON" ]; do + echo "⚠️ Sorry, that runtime environment is not yet supported. Only \"nodejs\" and \"python\" are currently supported." + read -p "🖥️ Enter the underlying runtime environment for \"$COMPONENT_PATH\" (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE + done +} + +# Helper function to remove an item from a list +removeFromList() { + ITEM_TO_REMOVE="$1" + NEW_LIST=() + for ITEM in "${UNIQUE_COMPONENTS_CLONE[@]}"; do + if [ "$ITEM" != "$ITEM_TO_REMOVE" ]; then + NEW_LIST+=("$ITEM") + fi + done + UNIQUE_COMPONENTS_CLONE=("${NEW_LIST[@]}") +} + +UNIQUE_COMPONENTS_CLONE=("${UNIQUE_COMPONENTS[@]}") +RUNTIME_LANGUAGE_DICT={} + +if [ "${MULTI_COMPONENT^^}" == 'Y' ]; then + if [ ! -z "$RUNTIME_LANGUAGE" ]; then # Environment Variable Passed + if echo "$RUNTIME_LANGUAGE" | jq -e > /dev/null 2>&1; then # Valid JSON + for key in $(echo "$RUNTIME_LANGUAGE" | jq -r 'keys[]'); do + removeFromList "$key" + done + if [ ${#UNIQUE_COMPONENTS_CLONE[@]} -gt 0 ]; then #if there are still components in the list, then not all runtimes were provided, so exit on error + echo "⚠️ You did not provide runtime languages for these components: \"${UNIQUE_COMPONENTS_CLONE[@]}\"." + exit 11 + fi + else + echo "⚠️ Your \"$RUNTIME_LANGUAGE\" is not valid JSON. Please re-format and try again." + exit 10 + fi + else # No Environment Variable Passed + for CURRENT in "${UNIQUE_COMPONENTS[@]}"; do + gatherRunTime "$CURRENT" + RUNTIME_LANGUAGE_DICT=$(echo "$RUNTIME_LANGUAGE_DICT" | jq --arg k "$CURRENT" --arg v "$RUNTIME_LANGUAGE" '. + {($k): $v}') + done + RUNTIME_LANGUAGE=$RUNTIME_LANGUAGE_DICT + echo "DICT: $RUNTIME_LANGUAGE_DICT" + fi +else + gatherRunTime "$PROJECT_REPOSITORY" +fi \ No newline at end of file diff --git a/deployment-scripts/gatherSetupCommands.sh b/deployment-scripts/gatherSetupCommands.sh new file mode 100644 index 00000000..44d5f0f7 --- /dev/null +++ b/deployment-scripts/gatherSetupCommands.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# This function gathers start up commands, such as build, install, and start, for both single and multiple component applications +# Last Modified by Maxwell Klema on July 15th, 2025 +# --------------------------------------------- + +gatherSetupCommands() { + + TYPE="$1" + PROMPT="$2" + TYPE_COMMAND="${TYPE}_COMMAND" + TYPE_COMMAND="${!TYPE_COMMAND}" # get value stored by TYPE_COMMAND + declare "COMMANDS_DICT={}" + + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + if [ ! -z "$TYPE_COMMAND" ]; then # Environment Variable Passed + if echo "$TYPE_COMMAND" | jq -e > /dev/null 2>&1; then # Valid JSON + for key in $(echo "$TYPE_COMMAND" | jq -r 'keys[]'); do + gatherComponentDir "Enter the path of your component to enter the ${TYPE,,} command" "$key" + addComponent "$key" + done + else + echo "⚠️ Your \"$TYPE_COMMAND\" is not valid JSON. Please re-format and try again." + exit 10 + fi + else # No Environment Variable Passed + gatherComponentDir "Enter the path of your component to enter the ${TYPE,,} command" + while [ "$COMPONENT_PATH" != "" ]; do + addComponent "$key" + read -p "$PROMPT" COMMAND + + # Append Component:Command k:v pair to map + COMMANDS_DICT=$(echo "$COMMANDS_DICT" | jq --arg k "$COMPONENT_PATH" --arg v "$COMMAND" '. + {($k): $v}') + gatherComponentDir "Enter the path of your component to enter the ${TYPE,,} command" + done + TYPE_COMMAND=$COMMANDS_DICT + fi + else + if [ -z "$TYPE_COMMAND" ]; then + read -p "$PROMPT" TYPE_COMMAND + fi + fi + + # Write to correct command variable + if [ "$TYPE" == "BUILD" ]; then + BUILD_COMMAND=$TYPE_COMMAND + elif [ "$TYPE" == "INSTALL" ]; then + INSTALL_COMMAND=$TYPE_COMMAND + else + START_COMMAND=$TYPE_COMMAND + fi +} \ No newline at end of file From 7bc7f92b1b171f24873fbdfbc8e4dbef344d140c Mon Sep 17 00:00:00 2001 From: maxklema Date: Wed, 16 Jul 2025 20:26:04 +0000 Subject: [PATCH 16/25] working on multi-app automatic deployment --- container creation/automaticLXCDeployment.sh | 144 +++++++++++++++---- container creation/create-container.sh | 112 +++++++++------ deployment-scripts/gatherBuildCommands.sh | 32 ----- deployment-scripts/gatherRuntimeLangs.sh | 6 +- deployment-scripts/gatherSetupCommands.sh | 4 +- 5 files changed, 184 insertions(+), 114 deletions(-) delete mode 100755 deployment-scripts/gatherBuildCommands.sh diff --git a/container creation/automaticLXCDeployment.sh b/container creation/automaticLXCDeployment.sh index 582f9b7f..17ffb923 100644 --- a/container creation/automaticLXCDeployment.sh +++ b/container creation/automaticLXCDeployment.sh @@ -1,73 +1,155 @@ #!/bin/bash # Automation Script for attempting to automatically deploy projects and services on a container -# Last Modifided by Maxwell Klema on July 11th, 2025 +# Last Modifided by Maxwell Klema on July 16th, 2025 # ----------------------------------------------------- echo "🚀 Attempting Automatic Deployment" REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") -# Clone github repository from correct branch +# Clone github repository from correct branch ==== pct enter $NEXT_ID < /dev/null 2>&1 EOF -# Copy over ENV variables -ENV_VARS=$(cat /var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE) -pct enter $NEXT_ID < .env +pct exec $NEXT_ID -- bash -c "chmod 700 ~/.bashrc" # enable full R/W/X permissions + +# Copy over ENV variables ==== + +ENV_BASE_FOLDER="/var/lib/vz/snippets/container-env-vars/${ENV_BASE_FOLDER}" + +if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then + for FILE in $ENV_BASE_FOLDER/*; do + FILE_BASENAME=$(basename "$FILE") + FILE_NAME="${FILE_BASENAME%.*}" + ENV_ROUTE=$(echo "$FILE_NAME" | tr '_' '/') # acts as the route to the correct folder to place .env file in. + + ENV_VARS=$(cat $ENV_BASE_FOLDER/$FILE_BASENAME) + pct enter $NEXT_ID < .env +EOF + done +else + ENV_FOLDER_BASE_NAME=$(basename "$ENV_BASE_FOLDER") + ENV_VARS=$(cat $ENV_BASE_FOLDER/$ENV_FOLDER_BASE_NAME.txt) + pct enter $NEXT_ID < .env EOF +fi + +# Run Installation Commands ==== + +runInstallCommands() { -# Install correct runtime and project dependencies -if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then - pct enter $NEXT_ID < /dev/null -sudo apt-get update -y && \ + RUNTIME="$1" + COMP_DIR="$2" + + if [ "${RUNTIME^^}" == "NODEJS" ]; then + if [ ${LINUX_DISTRO^^} == "DEBIAN" ]; then + pct enter $NEXT_ID < /dev/null +sudo apt update -y && \ sudo apt install -y nodejs npm && \ npm install -g pm2 && \ -cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && sudo $INSTALL_COMMAND +cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && sudo $INSTALL_CMD +EOF + elif [ "${LINUX_DISTRO^^}" == "ROCKY" ]; then + pct enter $NEXT_ID < /dev/null +sudo $PACKAGE_MANAGER install -y curl +curl -fsSL https://rpm.nodesource.com/setup_20.x | sudo bash - +sudo $PACKAGE_MANAGER update -y && \ +sudo $PACKAGE_MANAGER install -y nodejs && \ +npm install -g pm2 && \ +cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && sudo $INSTALL_CMD EOF -elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then - pct enter $NEXT_ID < /dev/null + fi + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + if [ ${LINUX_DISTRO^^} == "DEBIAN" ]; then + pct enter $NEXT_ID < /dev/null sudo apt install -y python3-venv python3-pip tmux && \ -cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && \ +cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && \ python3 -m venv venv && source venv/bin/activate && \ pip install --upgrade pip && \ -$INSTALL_COMMAND +$INSTALL_CMD EOF + elif [ "${LINUX_DISTRO^^}" == "ROCKY" ]; then + pct enter $NEXT_ID < /dev/null +sudo $PACKAGE_MANAGER install -y python python3-pip tmux && \ +cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && \ +python3 -m venv venv && source venv/bin/activate && \ +pip install --upgrade pip && \ +$INSTALL_CMD +EOF + fi + fi +} + +if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then + for COMPONENT in $(echo "$RUNTIME_LANGUAGE" | jq -r 'keys[]'); do + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') #get runtime env + INSTALL_CMD=$(echo "$INSTALL_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') #get install command + if [ "$INSTALL_CMD" != "null" ]; then + runInstallCommands "$RUNTIME" "$COMPONENT" + fi + done +else + INSTALL_CMD=$INSTALL_COMMAND + runInstallCommands "$RUNTIME_LANGUAGE" "." fi -# Iterate over each service installation command +# Install Services ==== + if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then while read line; do - pct exec $NEXT_ID -- bash -c "$line" > /dev/null + pct exec $NEXT_ID -- bash -c "$line" done < "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" fi -# Build Project (If Needed) and Start it +# Build and Start Processes ==== -if (( $NEXT_ID % 2 == 1)); then -pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 > /dev/null #temporarily bump up container resources for computation hungry processes (e.g. meteor)\ +startComponent() { - if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" - if [ "$BUILD_COMMAND" == "" ]; then - pct exec $NEXT_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && pm2 start bash -- -c '$START_COMMAND'" > /dev/null + if [ "${RUNTIME^^}" == "NODEJS" ]; then + if [ "$BUILD_CMD" == "" ]; then + pct exec $NEXT_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD'" > /dev/null else pct enter $NEXT_ID < /dev/null -cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && \ export PATH=\$PATH:/usr/local/bin && \ -$BUILD_COMMAND && pm2 start bash -- -c '$START_COMMAND' +$BUILD_CMD && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD' EOF fi - elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then - - if [ "$BUILD_COMMAND" == "" ]; then - pct exec $NEXT_ID -- "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $START_COMMAND" > /dev/null + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + if [ "$BUILD_CMD" == "" ]; then + pct exec $NEXT_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'" else - pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate $BUILD_COMMAND && $START_COMMAND'\" /dev/null + pct exec $NEXT_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'" /dev/null fi fi -pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 > /dev/null +} + +if (( $NEXT_ID % 2 == 1 )); then + pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 > /dev/null #temporarily bump up container resources for computation hungry processes (e.g. meteor) + + if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + + startComponent "$RUNTIME" "$BUILD" "$START" "$COMPONENT" + done + else + startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + fi + + pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 > /dev/null fi diff --git a/container creation/create-container.sh b/container creation/create-container.sh index a28485a7..d65c3dfc 100644 --- a/container creation/create-container.sh +++ b/container creation/create-container.sh @@ -1,6 +1,7 @@ #!/bin/bash # Script to create the pct container, run register container, and migrate container accordingly. -# Last Modified by July 11th, 2025 by Maxwell Klema +# Last Modified by July 16th, 2025 by Maxwell Klema +# ----------------------------------------------------- trap cleanup SIGINT SIGTERM SIGHUP @@ -20,8 +21,18 @@ INSTALL_COMMAND="${11}" BUILD_COMMAND="${12}" START_COMMAND="${13}" RUNTIME_LANGUAGE="${14}" -ENV_BASE_FILE="${15}" +ENV_BASE_FOLDER="${15}" SERVICES_BASE_FILE="${16}" +LINUX_DISTRO="${17}" +MULTI_COMPONENTS="${18}" + +if [ ${LINUX_DISTRO^^} == "DEBIAN" ]; then + PACKAGE_MANAGER="apt" + CTID_TEMPLATE="114" +elif [ "${LINUX_DISTRO^^}" == "ROCKY" ]; then + PACKAGE_MANAGER="dnf" + CTID_TEMPLATE="113" +fi REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") NEXT_ID=$(pvesh get /cluster/nextid) #Get the next available LXC ID @@ -36,15 +47,15 @@ function cleanup() echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" echo "⚠️ Script was abruptly exited. Running cleanup tasks." echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" - pct unlock 114 + pct unlock $CTID_TEMPLATE if [ -f "/var/lib/vz/snippets/container-public-keys/$PUB_FILE" ]; then rm -rf /var/lib/vz/snippets/container-public-keys/$PUB_FILE fi if [ -f "/var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE" ]; then rm -rf /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE fi - if [ -f "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE" ]; then - rm -rf "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE" + if [ -f "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" ]; then + rm -rf "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" fi if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then rm -rf "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" @@ -57,13 +68,13 @@ function cleanup() # Create the Container Clone echo "⏳ Cloning Container..." -pct clone 114 $NEXT_ID \ +pct clone $CTID_TEMPLATE $NEXT_ID \ --hostname $CONTAINER_NAME \ --full true > /dev/null 2>&1 # Set Container Options -echo "⏳ Setting Container Properties.." +echo "⏳ Setting Container Properties..." pct set $NEXT_ID \ --tags "$PROXMOX_USERNAME" \ --onboot 1 > /dev/null 2>&1 @@ -77,9 +88,11 @@ pveum aclmod /vms/$NEXT_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser > /de echo "⏳ Waiting for DHCP to allocate IP address to container..." sleep 10 +echo "⏳ Updating Packages.." + CONTAINER_IP=$(pct exec $NEXT_ID -- hostname -I | awk '{print $1}') -pct exec $NEXT_ID -- apt-get upgrade > /dev/null 2>&1 -pct exec $NEXT_ID -- apt install -y sudo git curl vim > /dev/null 2>&1 +pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER upgrade -y" > /dev/null +pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER install -y sudo git curl vim" > /dev/null if [ -f "/var/lib/vz/snippets/container-public-keys/$PUB_FILE" ]; then pct exec $NEXT_ID -- touch ~/.ssh/authorized_keys > /dev/null 2>&1 pct exec $NEXT_ID -- bash -c "cat > ~/.ssh/authorized_keys"< /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 @@ -93,11 +106,11 @@ pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev # Attempt to Automatically Deploy Project Inside Container if [ "${DEPLOY_ON_START^^}" == "Y" ]; then - source /var/lib/vz/snippets/helper-scripts/deployOnStart.sh + source /var/lib/vz/snippets/helper-scripts/clone-dos.sh #cleanup - if [ -f "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE" ]; then - rm -rf "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FILE" > /dev/null 2>&1 + if [ -f "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" ]; then + rm -rf "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" > /dev/null 2>&1 fi if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then rm -rf "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" > /dev/null 2>&1 @@ -115,48 +128,53 @@ fi SSH_PORT=$(iptables -t nat -S PREROUTING | grep "to-destination $CONTAINER_IP:22" | awk -F'--dport ' '{print $2}' | awk '{print $1}' | head -n 1 || true) -# Migrate to pve2 if Container ID is even +# Migrate to pve2 if Container ID is even and restart project ==== startProject() { -if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then - if [ "$BUILD_COMMAND" == "" ]; then - ssh root@10.15.0.5 " - pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && - pct exec $NEXT_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && pm2 start bash -- -c \"$START_COMMAND\"' && - pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 - " > /dev/null 2>&1 - else - ssh root@10.15.0.5 " - pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && - pct exec $NEXT_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && export PATH=\$PATH:/usr/local/bin && $BUILD_COMMAND && pm2 start bash -- -c \"$START_COMMAND\"' && - pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 - " > /dev/null 2>&1 + + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" + + if [ "${RUNTIME^^}" == "NODEJS" ]; then + if [ "$BUILD_CMD" == "" ]; then + ssh root@10.15.0.5 "pct exec $NEXT_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 + else + ssh root@10.15.0.5 "pct exec $NEXT_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && $BUILD_CMD && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 + fi + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + if [ "$BUILD_CMD" == "" ]; then + ssh root@10.15.0.5 "pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\"" > /dev/null 2>&1 + else + ssh root@10.15.0.5 "pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 + fi fi -elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then - if [ "$BUILD_COMMAND" == "" ]; then - ssh -T root@10.15.0.5 " - pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && \ - pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate && $START_COMMAND'\" /dev/null && \ - pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 - " > /dev/null 2>&1 - else - ssh root@10.15.0.5 " - pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 && \ - pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate $BUILD_COMMAND && $START_COMMAND'\" /dev/null && \ - pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 - " > /dev/null 2>&1 - fi -fi } if (( $NEXT_ID % 2 == 0 )); then - pct stop $NEXT_ID > /dev/null 2>&1 - pct migrate $NEXT_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 - ssh root@10.15.0.5 "pct start $NEXT_ID" > /dev/null 2>&1 - if [ "${DEPLOY_ON_START^^}" == "Y" ]; then - startProject - fi + pct stop $NEXT_ID > /dev/null 2>&1 + pct migrate $NEXT_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 + ssh root@10.15.0.5 "pct start $NEXT_ID" > /dev/null 2>&1 + ssh root@10.15.0.5 "pct exec $NEXT_ID -- bash -c 'chmod 700 ~/.bashrc'" # enable full R/W/X permissions + ssh root@10.15.0.5 "pct set $NEXT_ID --memory 4096 --swap 0 --cores 4" + if [ "${DEPLOY_ON_START^^}" == "Y" ]; then + if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" + done + else + startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + fi + fi + ssh root@10.15.0.5 "pct set $NEXT_ID --memory 2048 --swap 0 --cores 2" fi # Echo Container Details diff --git a/deployment-scripts/gatherBuildCommands.sh b/deployment-scripts/gatherBuildCommands.sh deleted file mode 100755 index 12230320..00000000 --- a/deployment-scripts/gatherBuildCommands.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -BUILD_COMMANDS_DICT={} - -if [ "${MULTI_COMPONENT^^}" == "Y" ]; then - if [ ! -z "$BUILD_COMMAND" ]; then # Environment Variable Passed - if echo "$BUILD_COMMAND" | jq -e > /dev/null 2>&1; then # Valid JSON - for key in $(echo "$BUILD_COMMAND" | jq -r 'keys[]'); do - gatherComponentDir "Enter the path of your component to enter the build command" "$key" - done - else - echo "⚠️ Your \"BUILD_COMMAND\" is not valid JSON. Please re-format and try again." - exit 10 - fi - else # No Environment Variable Passed - gatherComponentDir "Enter the path of your component to enter the build command" - while [ "$COMPONENT_PATH" != "" ]; do - read -p "🏗️ Enter the build command for \"$COMPONENT_PATH\" → " B_COMMAND - - # Append Component:Build_Command k:v pair to map - BUILD_COMMANDS_DICT=$(echo "$BUILD_COMMANDS_DICT" | jq --arg k "$COMPONENT_PATH" --arg v "$B_COMMAND" '. + {($k): $v}') - gatherComponentDir "Enter the path of your component to enter the build command" - done - BUILD_COMMAND=$BUILD_COMMANDS_DICT - fi -else - if [ -z "$BUILD_COMMAND" ]; then - read -p "🏗️ Enter the build command (Press Enter for no Build Command) → " BUILD_COMMAND - fi -fi - -echo "BUILD_COMMAND: $BUILD_COMMAND" - diff --git a/deployment-scripts/gatherRuntimeLangs.sh b/deployment-scripts/gatherRuntimeLangs.sh index 9ed7cae9..1331610f 100644 --- a/deployment-scripts/gatherRuntimeLangs.sh +++ b/deployment-scripts/gatherRuntimeLangs.sh @@ -3,7 +3,7 @@ gatherRunTime() { COMPONENT_PATH="$1" - if [ -z "${RUNTIME_LANGUAGE}" ] && [ "${MULTI_COMPONENT^^}" == "n" ]; then + if [ -z "${RUNTIME_LANGUAGE}" ] || [ "$RT_ENV_VAR" != "true" ]; then read -p "🖥️ Enter the underlying runtime environment for \"$COMPONENT_PATH\" (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE fi @@ -48,8 +48,10 @@ if [ "${MULTI_COMPONENT^^}" == 'Y' ]; then RUNTIME_LANGUAGE_DICT=$(echo "$RUNTIME_LANGUAGE_DICT" | jq --arg k "$CURRENT" --arg v "$RUNTIME_LANGUAGE" '. + {($k): $v}') done RUNTIME_LANGUAGE=$RUNTIME_LANGUAGE_DICT - echo "DICT: $RUNTIME_LANGUAGE_DICT" fi else + if [ ! -z "$RUNTIME_LANGUAGE" ]; then + RUNTIME_LANGUAGE="true" + fi gatherRunTime "$PROJECT_REPOSITORY" fi \ No newline at end of file diff --git a/deployment-scripts/gatherSetupCommands.sh b/deployment-scripts/gatherSetupCommands.sh index 44d5f0f7..4aaf6478 100644 --- a/deployment-scripts/gatherSetupCommands.sh +++ b/deployment-scripts/gatherSetupCommands.sh @@ -1,6 +1,6 @@ #!/bin/bash # This function gathers start up commands, such as build, install, and start, for both single and multiple component applications -# Last Modified by Maxwell Klema on July 15th, 2025 +# Last Modified by Maxwell Klema on July 16th, 2025 # --------------------------------------------- gatherSetupCommands() { @@ -25,7 +25,7 @@ gatherSetupCommands() { else # No Environment Variable Passed gatherComponentDir "Enter the path of your component to enter the ${TYPE,,} command" while [ "$COMPONENT_PATH" != "" ]; do - addComponent "$key" + addComponent "$COMPONENT_PATH" read -p "$PROMPT" COMMAND # Append Component:Command k:v pair to map From b633d18b464eff873b9d04b31edfcd752fcfbb7c Mon Sep 17 00:00:00 2001 From: maxklema Date: Thu, 17 Jul 2025 17:08:31 +0000 Subject: [PATCH 17/25] root start command support + misc fixes --- container creation/automaticLXCDeployment.sh | 21 +- container creation/create-container.sh | 13 +- container creation/get-deployment-details.sh | 118 ++--- .../get-lxc-container-details.sh | 500 ++++++++++-------- container maintenance/update-container.sh | 137 ++--- deployment-scripts/gatherEnvVars.sh | 5 +- 6 files changed, 418 insertions(+), 376 deletions(-) diff --git a/container creation/automaticLXCDeployment.sh b/container creation/automaticLXCDeployment.sh index 17ffb923..4829902b 100644 --- a/container creation/automaticLXCDeployment.sh +++ b/container creation/automaticLXCDeployment.sh @@ -11,7 +11,7 @@ REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") pct enter $NEXT_ID < /dev/null 2>&1 +git clone --branch $PROJECT_BRANCH --single-branch $PROJECT_REPOSITORY EOF pct exec $NEXT_ID -- bash -c "chmod 700 ~/.bashrc" # enable full R/W/X permissions @@ -49,8 +49,10 @@ runInstallCommands() { if [ "${RUNTIME^^}" == "NODEJS" ]; then if [ ${LINUX_DISTRO^^} == "DEBIAN" ]; then pct enter $NEXT_ID < /dev/null -sudo apt update -y && \ -sudo apt install -y nodejs npm && \ +sudo $PACKAGE_MANAGER install -y curl +curl -fsSL https://deb.nodesource.com/setup_20.x | sudo bash - +sudo $PACKAGE_MANAGER update -y && \ +sudo $PACKAGE_MANAGER install -y nodejs && \ npm install -g pm2 && \ cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && sudo $INSTALL_CMD EOF @@ -102,7 +104,7 @@ fi if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then while read line; do - pct exec $NEXT_ID -- bash -c "$line" + pct exec $NEXT_ID -- bash -c "$line" > /dev/null 2>&1 done < "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" fi @@ -117,7 +119,7 @@ startComponent() { if [ "${RUNTIME^^}" == "NODEJS" ]; then if [ "$BUILD_CMD" == "" ]; then - pct exec $NEXT_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD'" > /dev/null + pct exec $NEXT_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD'" > /dev/null 2>&1 else pct enter $NEXT_ID < /dev/null export PATH=\$PATH:/usr/local/bin && \ @@ -126,16 +128,15 @@ EOF fi elif [ "${RUNTIME^^}" == "PYTHON" ]; then if [ "$BUILD_CMD" == "" ]; then - pct exec $NEXT_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'" + pct exec $NEXT_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'" > /dev/null 2>&1 else - pct exec $NEXT_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'" /dev/null + pct exec $NEXT_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'" > /dev/null 2>&1 fi fi } if (( $NEXT_ID % 2 == 1 )); then pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 > /dev/null #temporarily bump up container resources for computation hungry processes (e.g. meteor) - if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') @@ -147,9 +148,11 @@ if (( $NEXT_ID % 2 == 1 )); then startComponent "$RUNTIME" "$BUILD" "$START" "$COMPONENT" done + if [ ! -z "$ROOT_START_COMMAND" ]; then + pct exec $NEXT_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" > /dev/null 2>&1 + fi else startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." fi - pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 > /dev/null fi diff --git a/container creation/create-container.sh b/container creation/create-container.sh index d65c3dfc..b0a3865e 100644 --- a/container creation/create-container.sh +++ b/container creation/create-container.sh @@ -1,7 +1,6 @@ #!/bin/bash # Script to create the pct container, run register container, and migrate container accordingly. -# Last Modified by July 16th, 2025 by Maxwell Klema -# ----------------------------------------------------- +# Last Modified by July 11th, 2025 by Maxwell Klema trap cleanup SIGINT SIGTERM SIGHUP @@ -25,6 +24,7 @@ ENV_BASE_FOLDER="${15}" SERVICES_BASE_FILE="${16}" LINUX_DISTRO="${17}" MULTI_COMPONENTS="${18}" +ROOT_START_COMMAND="${19}" if [ ${LINUX_DISTRO^^} == "DEBIAN" ]; then PACKAGE_MANAGER="apt" @@ -106,7 +106,7 @@ pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev # Attempt to Automatically Deploy Project Inside Container if [ "${DEPLOY_ON_START^^}" == "Y" ]; then - source /var/lib/vz/snippets/helper-scripts/clone-dos.sh + source /var/lib/vz/snippets/helper-scripts/deployOnStart.sh #cleanup if [ -f "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" ]; then @@ -157,8 +157,8 @@ if (( $NEXT_ID % 2 == 0 )); then pct stop $NEXT_ID > /dev/null 2>&1 pct migrate $NEXT_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 ssh root@10.15.0.5 "pct start $NEXT_ID" > /dev/null 2>&1 - ssh root@10.15.0.5 "pct exec $NEXT_ID -- bash -c 'chmod 700 ~/.bashrc'" # enable full R/W/X permissions - ssh root@10.15.0.5 "pct set $NEXT_ID --memory 4096 --swap 0 --cores 4" + ssh root@10.15.0.5 "pct exec $NEXT_ID -- bash -c 'chmod 700 ~/.bashrc'" > /dev/null 2>&1 # enable full R/W/X permissions + ssh root@10.15.0.5 "pct set $NEXT_ID --memory 4096 --swap 0 --cores 4" > /dev/null 2>&1 if [ "${DEPLOY_ON_START^^}" == "Y" ]; then if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do @@ -170,6 +170,9 @@ if (( $NEXT_ID % 2 == 0 )); then fi startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" done + if [ ! -z "$ROOT_START_COMMAND" ]; then + ssh root@10.15.0.5 "pct exec $NEXT_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi else startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." fi diff --git a/container creation/get-deployment-details.sh b/container creation/get-deployment-details.sh index b0975c52..5d1e9631 100755 --- a/container creation/get-deployment-details.sh +++ b/container creation/get-deployment-details.sh @@ -1,6 +1,6 @@ #!/bin/bash # Helper script to gather project details for automatic deployment -# Modified July 9th, 2025 by Maxwell Klema +# Modified July 17th, 2025 by Maxwell Klema # ------------------------------------------ # Define color variables (works on both light and dark backgrounds) @@ -71,93 +71,77 @@ if [[ "$PROJECT_ROOT" == "/*" ]]; then PROJECT_ROOT="${PROJECT_ROOT:1}" fi -# Get Environment Variables ======== +# Check if the App has multiple components (backend, frontend, multiple servers, etc.) ======== -gatherEnvVars(){ - - read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY - read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE - - while [ "$ENV_VAR_KEY" == "" ] || [ "$ENV_VAR_VALUE" == "" ]; do - echo "⚠️ Key or value cannot be empty. Try again." - read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY - read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE - done - - echo "$ENV_VAR_KEY=$ENV_VAR_VALUE" >> $TEMP_ENV_FILE_PATH - - read -p "🔑 Do you want to enter another Environment Variable? (y/n) → " ENTER_ANOTHER_ENV -} - -if [ -z "$REQUIRE_ENV_VARS" ]; then - read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS +if [ -z "$MULTI_COMPONENT" ]; then + read -p "🔗 Does your app consist of multiple components that run independently, i.e. seperate frontend and backend (y/n) → " MULTI_COMPONENT fi -while [ "${REQUIRE_ENV_VARS^^}" != "Y" ] && [ "${REQUIRE_ENV_VARS^^}" != "N" ] && [ "${REQUIRE_ENV_VARS^^}" != "" ]; do - echo "⚠️ Invalid option. Please try again." - read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS +while [ "${MULTI_COMPONENT^^}" != "Y" ] && [ "${MULTI_COMPONENT^^}" != "N" ] && [ "${MULTI_COMPONENT^^}" != "" ]; do + echo "⚠️ Invalid option. Please try again." + read -p "🔗 Does your app consist of multiple components that run independently, i.e. seperate frontend and backend (y/n) → " MULTI_COMPONENT done -if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then +# Gather Deployment Commands ======== - # generate random temp .env file - RANDOM_NUM=$(shuf -i 100000-999999 -n 1) - ENV_FILE="env_$RANDOM_NUM.txt" - TEMP_ENV_FILE_PATH="/root/bin/env/$ENV_FILE" - touch "$TEMP_ENV_FILE_PATH" +# Helper functions to gather and validate component directory +gatherComponentDir() { - if [ ! -z "$CONTAINER_ENV_VARS" ]; then - if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success - echo "$CONTAINER_ENV_VARS " | jq -r 'to_entries[] | "\(.key)=\(.value)"' > "$TEMP_ENV_FILE_PATH" #k=v pairs + COMPONENT_PATH="$2" + if [ -z "$COMPONENT_PATH" ]; then + read -p "$1, relative to project root directory (To Continue, Press Enter) → " COMPONENT_PATH + fi + # Check that component path is valid + VALID_COMPONENT_PATH=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$COMPONENT_PATH") + while [ "$VALID_COMPONENT_PATH" == "false" ] && [ "$COMPONENT_PATH" != "" ]; do + echo "⚠️ The component path you entered, \"$COMPONENT_PATH\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." + if [ -z "$2" ]; then + read -p "$1, relative to project root directory (To Continue, Press Enter) → " COMPONENT_PATH + VALID_COMPONENT_PATH=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$COMPONENT_PATH") else - echo "⚠️ Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." - exit 10 + exit 9 fi - else - gatherEnvVars - while [ "${ENTER_ANOTHER_ENV^^}" == "Y" ]; do - gatherEnvVars - done + done + + if [[ "$COMPONENT_PATH" == /* ]]; then + COMPONENT_PATH="${COMPONENT_PATH:1}" # remove leading slash fi -fi +} -# Get Install Command ======== +UNIQUE_COMPONENTS=() -if [ -z "$INSTALL_COMMAND" ]; then - read -p "📦 Enter the install command (e.g., 'npm install') → " INSTALL_COMMAND -fi +# Helper function to add a component to unique components if its not already present +addComponent() { + COMPONENT="$1" + for CURRENT in "${UNIQUE_COMPONENTS[@]}"; do + if [ "${COMPONENT,,}" == "${CURRENT,,}" ]; then + return 0 + fi + done + UNIQUE_COMPONENTS+=("$COMPONENT") +} -# Get Build Command ======== +source /root/bin/deployment-scripts/gatherSetupCommands.sh # Function to gather build, install, and start commands -if [ -z "$BUILD_COMMAND" ]; then - read -p "🏗️ Enter the build command (leave blank if no build command) → " BUILD_COMMAND -fi +source /root/bin/deployment-scripts/gatherEnvVars.sh # Gather Environment Variables +gatherSetupCommands "BUILD" "🏗️ Enter the build command (leave blank if no build command) → " # Gather Build Command(s) +gatherSetupCommands "INSTALL" "📦 Enter the install command (e.g., 'npm install') → " # Gather Install Command(s)echo "$INSTALL_COMMAND" +gatherSetupCommands "START" "🚦 Enter the start command (e.g., 'npm start', 'python app.py') → " # Gather Start Command(s) -# Get Start Command ======== -if [ -z "$START_COMMAND" ]; then - read -p "🚦 Enter the start command (e.g., 'npm start', 'python app.py') → " START_COMMAND +if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + if [ -z "$ROOT_START_COMMAND" ]; then + read -p "📍 If your container requires a start command at the root directory, i.e. Docker run, enter it here (leave blank for no command) → " ROOT_START_COMMAND + fi fi -while [ "$START_COMMAND" == "" ]; do - echo "⚠️ The start command cannot be blank. Please try again." - read -p "🚦 Enter the start command (e.g., 'npm start') → " START_COMMAND -done - # Get Runtime Language ======== -if [ -z "$RUNTIME_LANGUAGE" ]; then - read -p "🖥️ Enter the underlying runtime environment (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE -fi - -while [ "${RUNTIME_LANGUAGE^^}" != "NODEJS" ] && [ "${RUNTIME_LANGUAGE^^}" != "PYTHON" ]; do - echo "⚠️ Sorry, that runtime environment is not yet supported. Only \"nodejs\" and \"python\" are currently supported." - read -p "🖥️ Enter the underlying runtime environment (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE -done +source /root/bin/deployment-scripts/gatherRuntimeLangs.sh # Get Services ======== -SERVICE_MAP="/root/bin/services/service_map.json" +SERVICE_MAP="/root/bin/services/service_map_$LINUX_DISTRIBUTION.json" APPENDED_SERVICES=() # Helper function to check if a user has added the same service twice @@ -240,7 +224,7 @@ appendCustomService() { # Helper function to see if a user wants to set up a custom service setUpService() { read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION - while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do + while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do echo "⚠️ Invalid option. Please try again." read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION done @@ -250,7 +234,7 @@ if [ -z "$REQUIRE_SERVICES" ]; then read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES fi -while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do +while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do echo "⚠️ Invalid option. Please try again." read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES done diff --git a/container creation/get-lxc-container-details.sh b/container creation/get-lxc-container-details.sh index b0975c52..3c746421 100644 --- a/container creation/get-lxc-container-details.sh +++ b/container creation/get-lxc-container-details.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Helper script to gather project details for automatic deployment -# Modified July 9th, 2025 by Maxwell Klema +# Main Container Creation Script +# Modified July 17th, 2025 by Maxwell Klema # ------------------------------------------ # Define color variables (works on both light and dark backgrounds) @@ -9,277 +9,315 @@ BOLD="\033[1m" MAGENTA='\033[35m' echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -echo -e "${BOLD}${MAGENTA}🌐 Let's Get Your Project Automatically Deployed ${RESET}" +echo -e "${BOLD}${MAGENTA}📦 MIE Container Creation Script ${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -# Get and validate project repository ======== +# Authenticate User (Only Valid Users can Create Containers) -if [ -z "$PROJECT_REPOSITORY" ]; then - read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY +if [ -z "$PROXMOX_USERNAME" ]; then + read -p "Enter Proxmox Username → " PROXMOX_USERNAME fi -CheckRepository() { - PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY#*github.com/} - PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY_SHORTENED%.git} - REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$RROJECT_REPOSITORY) -} - -CheckRepository +if [ -z "$PROXMOX_PASSWORD" ]; then + read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD + echo "" +fi -while [ "$REPOSITORY_EXISTS" != "200" ]; do - echo "⚠️ The repository link you provided, \"$PROJECT_REPOSITORY\" was not valid." - read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY - CheckRepository +USER_AUTHENTICATED=$(node /root/bin/js/runner.js authenticateUser "$PROXMOX_USERNAME" "$PROXMOX_PASSWORD") +RETRIES=3 + +while [ $USER_AUTHENTICATED == 'false' ]; do + if [ $RETRIES -gt 0 ]; then + echo "❌ Authentication Failed. Try Again" + read -p "Enter Proxmox Username → " PROXMOX_USERNAME + read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD + echo "" + + USER_AUTHENTICATED=$(node /root/bin/js/runner.js authenticateUser "$PROXMOX_USERNAME" "$PROXMOX_PASSWORD") + RETRIES=$(($RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 2 + fi done -# Get Repository Branch ======== +echo "🎉 Your proxmox account, $PROXMOX_USERNAME@pve, has been authenticated" -if [ -z "$PROJECT_BRANCH" ]; then - read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH -fi +# Gather Container Hostname (hostname.opensource.mieweb.org) -if [ "$PROJECT_BRANCH" == "" ]; then - PROJECT_BRANCH="main" +if [ -z "$CONTAINER_NAME" ]; then + read -p "Enter Application Name (One-Word) → " CONTAINER_NAME fi -REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY/tree/$PROJECT_BRANCH) -while [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; do - echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." - read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH - if [ "PROJECT_BRANCH" == "" ]; then - PROJECT_BRANCH="main" - fi - REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY_SHORTENED/tree/$PROJECT_BRANCH) +CONTAINER_NAME="${CONTAINER_NAME,,}" #convert to lowercase +HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") +HOST_NAME_RETRIES=10 + +while [[ $HOST_NAME_EXISTS == 'true' ]] || ! [[ "$CONTAINER_NAME" =~ ^[A-Za-z0-9-]+$ ]]; do + if [ $HOST_NAME_RETRIES -gt 0 ]; then + echo "Sorry! Either that name has already been registered or your hostname is ill-formatted. Try another name" + read -p "Enter Application Name (One-Word) → " CONTAINER_NAME + HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") + HOST_NAME_RETRIES=$(($HOST_NAME_RETRIES-1)) + CONTAINER_NAME="${CONTAINER_NAME,,}" + else + echo "Too many incorrect attempts. Exiting..." + exit 3 + fi done -# Get Project Root Directory ======== - -if [ -z "$PROJECT_ROOT" ]; then - read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT +echo "✅ $CONTAINER_NAME is available" + +# Gather Container Password +PASSWORD_RETRIES=10 + +if [ -z "$CONTAINER_PASSWORD" ]; then + read -sp "Enter Container Password → " CONTAINER_PASSWORD + echo + read -sp "Confirm Container Password → " CONFIRM_PASSWORD + echo + + while [[ "$CONFIRM_PASSWORD" != "$CONTAINER_PASSWORD" || ${#CONTAINER_PASSWORD} -lt 8 ]]; do + if [ $PASSWORD_RETRIES -gt 0 ]; then + echo "Sorry, try again. Ensure passwords are at least 8 characters." + read -sp "Enter Container Password → " CONTAINER_PASSWORD + echo + read -sp "Confirm Container Password → " CONFIRM_PASSWORD + echo + PASSWORD_RETRIES=$(($PASSWORD_RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 4 + fi + done +else + CONFIRM_PASSWORD="$CONTAINER_PASSWORD" + while [[ "$CONFIRM_PASSWORD" != "$CONTAINER_PASSWORD" || ${#CONTAINER_PASSWORD} -lt 8 ]]; do + if [ $PASSWORD_RETRIES -gt 0 ]; then + echo "Sorry, try again. Ensure passwords are at least 8 characters." + read -sp "Enter Container Password → " CONTAINER_PASSWORD + echo + read -sp "Confirm Container Password → " CONFIRM_PASSWORD + echo + PASSWORD_RETRIES=$(($PASSWORD_RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 4 + fi + done fi -VALID_PROJECT_ROOT=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") - -while [ "$VALID_PROJECT_ROOT" == "false" ]; do - echo "⚠️ The root directory you provided, \"$PROJECT_ROOT\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." - read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT - VALID_PROJECT_ROOT=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") -done +# Choose Linux Distribution -# Remove forward slash -if [[ "$PROJECT_ROOT" == "/*" ]]; then - PROJECT_ROOT="${PROJECT_ROOT:1}" +if [ -z "$LINUX_DISTRIBUTION" ]; then + echo "🐧 Available Linux Distributions:" + echo "1. Debian 12 (Bookworm)" + echo "2. Rocky 9 " + read -p "➡️ Choose a Linux Distribution (debian/rocky) → " LINUX_DISTRIBUTION fi -# Get Environment Variables ======== - -gatherEnvVars(){ - - read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY - read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE +while [ "${LINUX_DISTRIBUTION,,}" != "debian" ] && [ "${LINUX_DISTRIBUTION,,}" != "rocky" ]; do + echo "❌ Please choose a valid Linux Distribution." + echo "1. Debian 12 (Bookworm)" + echo "2. Rocky 9 " + read -p "➡️ Choose a Linux Distribution (debian/rocky) → " LINUX_DISTRIBUTION +done - while [ "$ENV_VAR_KEY" == "" ] || [ "$ENV_VAR_VALUE" == "" ]; do - echo "⚠️ Key or value cannot be empty. Try again." - read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY - read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE - done +LINUX_DISTRIBUTION=${LINUX_DISTRIBUTION,,} + +# Attempt to detect public keys + +echo -e "\n🔑 Attempting to Detect SSH Public Key..." + +AUTHORIZED_KEYS="/root/.ssh/authorized_keys" +RANDOM_NUM=$(shuf -i 100000-999999 -n 1) +PUB_FILE="key_$RANDOM_NUM.pub" +TEMP_PUB_FILE="/root/bin/ssh/temp_pubs/$PUB_FILE" # in case two users are running this script at the same time, they do not overwrite each other's temp files +touch "$TEMP_PUB_FILE" +DETECT_PUBLIC_KEY=$(sudo /root/bin/ssh/detectPublicKey.sh "$SSH_KEY_FP" "$TEMP_PUB_FILE") +KEY_RETRIES=10 + +if [ "$DETECT_PUBLIC_KEY" == "Public key found for create-container" ]; then + echo "🔐 Public Key Found!" +else + echo "🔍 Could not detect Public Key" + + if [ -z "$PUBLIC_KEY" ]; then + read -p "Enter Public Key (Allows Easy Access to Container) [OPTIONAL - LEAVE BLANK TO SKIP] → " PUBLIC_KEY + fi + + # Check if key is valid + + while [[ "$PUBLIC_KEY" != "" && $(echo "$PUBLIC_KEY" | ssh-keygen -l -f - 2>&1 | tr -d '\r') == "(stdin) is not a public key file." ]]; do + if [ $KEY_RETRIES -gt 0 ]; then + echo "❌ \"$PUBLIC_KEY\" is not a valid key. Enter either a valid key or leave blank to skip." + read -p "Enter Public Key (Allows Easy Access to Container) [OPTIONAL - LEAVE BLANK TO SKIP] → " PUBLIC_KEY + KEY_RETRIES=$(($KEY_RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 5 + fi + done + + if [ "$PUBLIC_KEY" != "" ]; then + echo "$PUBLIC_KEY" > "$AUTHORIZED_KEYS" && systemctl restart ssh + echo "$PUBLIC_KEY" > "$TEMP_PUB_FILE" + sudo /root/bin/ssh/publicKeyAppendJumpHost.sh "$PUBLIC_KEY" + fi +fi - echo "$ENV_VAR_KEY=$ENV_VAR_VALUE" >> $TEMP_ENV_FILE_PATH +# Get HTTP Port Container Listens On +HTTP_PORT_RETRIES=10 - read -p "🔑 Do you want to enter another Environment Variable? (y/n) → " ENTER_ANOTHER_ENV -} - -if [ -z "$REQUIRE_ENV_VARS" ]; then - read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS +if [ -z "$HTTP_PORT" ]; then + read -p "Enter HTTP Port for your container to listen on (80-60000) → " HTTP_PORT fi -while [ "${REQUIRE_ENV_VARS^^}" != "Y" ] && [ "${REQUIRE_ENV_VARS^^}" != "N" ] && [ "${REQUIRE_ENV_VARS^^}" != "" ]; do - echo "⚠️ Invalid option. Please try again." - read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS +while ! [[ "$HTTP_PORT" =~ ^[0-9]+$ ]] || [ "$HTTP_PORT" -lt 80 ] || [ "$HTTP_PORT" -gt 60000 ]; do + if [ $HTTP_PORT_RETRIES -gt 0 ]; then + echo "❌ Invalid HTTP Port. It must be a number between 80 and 60,000." + read -p "Enter HTTP Port for your container to listen on (80-60000) → " HTTP_PORT + HTTP_PORT_RETRIES=$(($HTTP_PORT_RETRIES-1)) + else + echo "Too many incorrect attempts. Exiting..." + exit 6 + fi done -if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then - - # generate random temp .env file - RANDOM_NUM=$(shuf -i 100000-999999 -n 1) - ENV_FILE="env_$RANDOM_NUM.txt" - TEMP_ENV_FILE_PATH="/root/bin/env/$ENV_FILE" - touch "$TEMP_ENV_FILE_PATH" - - if [ ! -z "$CONTAINER_ENV_VARS" ]; then - if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success - echo "$CONTAINER_ENV_VARS " | jq -r 'to_entries[] | "\(.key)=\(.value)"' > "$TEMP_ENV_FILE_PATH" #k=v pairs - else - echo "⚠️ Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." - exit 10 - fi - else - gatherEnvVars - while [ "${ENTER_ANOTHER_ENV^^}" == "Y" ]; do - gatherEnvVars - done - fi -fi - -# Get Install Command ======== +echo "✅ HTTP Port is set to $HTTP_PORT" -if [ -z "$INSTALL_COMMAND" ]; then - read -p "📦 Enter the install command (e.g., 'npm install') → " INSTALL_COMMAND -fi +# Get any other protocols -# Get Build Command ======== +protocol_duplicate() { + PROTOCOL="$1" + shift #remaining params are part of list + LIST="$@" -if [ -z "$BUILD_COMMAND" ]; then - read -p "🏗️ Enter the build command (leave blank if no build command) → " BUILD_COMMAND -fi + for item in $LIST; do + if [[ "$item" == "$PROTOCOL" ]]; then + return 0 # Protocol is a duplicate + fi + done + return 1 # Protocol is not a duplicate +} -# Get Start Command ======== +read -p "Does your Container require any protocols other than SSH and HTTP? (y/n) → " USE_OTHER_PROTOCOLS +while [ "${USE_OTHER_PROTOCOLS^^}" != "Y" ] && [ "${USE_OTHER_PROTOCOLS^^}" != "N" ] && [ "${USER_OTHER_PROTOCOLS^^}" != "" ]; do + echo "Please answer 'y' for yes or 'n' for no." + read -p "Does your Container require any protocols other than SSH and HTTP? (y/n) → " USE_OTHER_PROTOCOLS +done -if [ -z "$START_COMMAND" ]; then - read -p "🚦 Enter the start command (e.g., 'npm start', 'python app.py') → " START_COMMAND +RANDOM_NUM=$(shuf -i 100000-999999 -n 1) +PROTOCOL_BASE_FILE="protocol_list_$RANDOM_NUM.txt" +PROTOCOL_FILE="/root/bin/protocols/$PROTOCOL_BASE_FILE" +touch "$PROTOCOL_FILE" + +if [ "${USE_OTHER_PROTOCOLS^^}" == "Y" ]; then + LIST_PROTOCOLS=() + read -p "Enter the protocol abbreviation (e.g, LDAP for Lightweight Directory Access Protocol). Type \"e\" to exit → " PROTOCOL_NAME + while [ "${PROTOCOL_NAME^^}" != "E" ]; do + FOUND=0 #keep track if protocol was found + while read line; do + PROTOCOL_ABBRV=$(echo "$line" | awk '{print $1}') + protocol_duplicate "$PROTOCOL_ABBRV" "${LIST_PROTOCOLS[@]}" + IS_PROTOCOL_DUPLICATE=$? + if [[ "$PROTOCOL_ABBRV" == "${PROTOCOL_NAME^^}" && "$IS_PROTOCOL_DUPLICATE" -eq 1 ]]; then + LIST_PROTOCOLS+=("$PROTOCOL_ABBRV") + PROTOCOL_UNDRLYING_NAME=$(echo "$line" | awk '{print $3}') + PROTOCOL_DEFAULT_PORT=$(echo "$line" | awk '{print $2}') + echo "$PROTOCOL_ABBRV $PROTOCOL_UNDRLYING_NAME $PROTOCOL_DEFAULT_PORT" >> "$PROTOCOL_FILE" + echo "✅ Protocol ${PROTOCOL_NAME^^} added to container." + FOUND=1 #protocol was found + break + else + echo "❌ Protocol ${PROTOCOL_NAME^^} was already added to your container. Please try again." + FOUND=2 #protocol was a duplicate + break + fi + done < <(cat "/root/bin/protocols/master_protocol_list.txt" | grep "^${PROTOCOL_NAME^^}") + + if [ $FOUND -eq 0 ]; then #if no results found, let user know. + echo "❌ Protocol ${PROTOCOL_NAME^^} not found. Please try again." + fi + + read -p "Enter the protocol abbreviation (e.g, LDAP for Lightweight Directory Access Protocol). Type \"e\" to exit → " PROTOCOL_NAME + done fi -while [ "$START_COMMAND" == "" ]; do - echo "⚠️ The start command cannot be blank. Please try again." - read -p "🚦 Enter the start command (e.g., 'npm start') → " START_COMMAND -done -# Get Runtime Language ======== +# Attempt to deploy application on start. -if [ -z "$RUNTIME_LANGUAGE" ]; then - read -p "🖥️ Enter the underlying runtime environment (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE +if [ -z "$DEPLOY_ON_START" ]; then + read -p "🚀 Do you want to deploy your project automatically? (y/n) → " DEPLOY_ON_START fi -while [ "${RUNTIME_LANGUAGE^^}" != "NODEJS" ] && [ "${RUNTIME_LANGUAGE^^}" != "PYTHON" ]; do - echo "⚠️ Sorry, that runtime environment is not yet supported. Only \"nodejs\" and \"python\" are currently supported." - read -p "🖥️ Enter the underlying runtime environment (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE +while [ "${DEPLOY_ON_START^^}" != "Y" ] && [ "${DEPLOY_ON_START^^}" != "N" ] && [ "${DEPLOY_ON_START^^}" != "" ]; do + echo "Please answer 'y' for yes or 'n' for no." + read -p "🚀 Do you want to deploy your project automatically? (y/n) → " DEPLOY_ON_START done -# Get Services ======== - -SERVICE_MAP="/root/bin/services/service_map.json" -APPENDED_SERVICES=() - -# Helper function to check if a user has added the same service twice -serviceExists() { - SERVICE="$1" - for CURRENT in "${APPENDED_SERVICES[@]}"; do - if [ "${SERVICE,,}" == "${CURRENT,,}" ]; then - return 0 - fi - done - return 1 -} - -processService() { - local SERVICE="$1" - local MODE="$2" # "batch" or "single" - - SERVICE_IN_MAP=$(jq -r --arg key "${SERVICE,,}" '.[$key] // empty' "$SERVICE_MAP") - if serviceExists "$SERVICE"; then - if [ "$MODE" = "batch" ]; then - return 0 # skip to next in batch mode - else - echo "⚠️ You already added \"$SERVICE\" as a service. Please try again." - return 0 - fi - elif [ "${SERVICE^^}" != "C" ] && [ "${SERVICE^^}" != "" ] && [ -n "$SERVICE_IN_MAP" ]; then - jq -r --arg key "${SERVICE,,}" '.[$key][]' "$SERVICE_MAP" >> "$TEMP_SERVICES_FILE_PATH" - echo "sudo systemctl daemon-reload" >> "$TEMP_SERVICES_FILE_PATH" - echo "✅ ${SERVICE^^} added to your container." - APPENDED_SERVICES+=("${SERVICE^^}") - elif [ "${SERVICE^^}" == "C" ]; then - appendCustomService - elif [ "${SERVICE^^}" != "" ]; then - echo "⚠️ Service \"$SERVICE\" does not exist." - [ "$MODE" = "batch" ] && exit 20 - fi -} - -# Helper function to append a new service to a container -appendService() { - if [ ! -z "$SERVICES" ]; then - for SERVICE in $(echo "$SERVICES" | jq -r '.[]'); do - processService "$SERVICE" "batch" - done - else - read -p "➡️ Enter the name of a service to add to your container or type \"C\" to set up a custom service installation (Enter to exit) → " SERVICE - processService "$SERVICE" "single" - fi -} +if [ "${DEPLOY_ON_START^^}" == "Y" ]; then + source /root/bin/deploy-application.sh +fi -appendCustomService() { - # If there is an env variable for custom services, iterate through each command and append it to temporary services file - if [ ! -z "$CUSTOM_SERVICES" ]; then - echo "$CUSTOM_SERVICES" | jq -c -r '.[]' | while read -r CUSTOM_SERVICE; do - echo "$CUSTOM_SERVICE" | jq -c -r '.[]' | while read -r CUSTOM_SERVICE_COMMAND; do - if [ ! -z "$CUSTOM_SERVICE_COMMAND" ]; then - echo "$CUSTOM_SERVICE_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" - else - echo "⚠️ Command cannot be empty." - exit 21; - fi - done - done - echo "✅ Custom Services appended." - else - echo "🛎️ Configuring Custom Service Installation. For each prompt, enter a command that is a part of the installation process for your service on Debian Bookworm. Do not forget to enable and start the service at the end. Once you have entered all of your commands, press enter to continue" - COMMAND_NUM=1 - read -p "➡️ Enter Command $COMMAND_NUM: " CUSTOM_COMMAND - - echo "$CUSTOM_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" - - while [ "${CUSTOM_COMMAND^^}" != "" ]; do - ((COMMAND_NUM++)) - read -p "➡️ Enter Command $COMMAND_NUM: " CUSTOM_COMMAND - echo "$CUSTOM_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" - done +# send public key, port mapping, env vars, and services to hypervisor + +send_file_to_hypervisor() { + local LOCAL_FILE="$1" + local REMOTE_FOLDER="$2" + if [ "$REMOTE_FOLDER" != "container-env-vars" ]; then + if [ -s "$LOCAL_FILE" ]; then + sftp root@10.15.0.4 < /dev/null +put $LOCAL_FILE /var/lib/vz/snippets/$REMOTE_FOLDER/ +EOF + fi + else + if [ -d "$LOCAL_FILE" ]; then + sftp root@10.15.0.4 < /dev/null +put -r $LOCAL_FILE /var/lib/vz/snippets/$REMOTE_FOLDER/ +EOF + else + ENV_FOLDER="null" + fi fi } -# Helper function to see if a user wants to set up a custom service -setUpService() { - read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION - while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do - echo "⚠️ Invalid option. Please try again." - read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION - done -} - -if [ -z "$REQUIRE_SERVICES" ]; then - read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES -fi - -while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do - echo "⚠️ Invalid option. Please try again." - read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES -done +send_file_to_hypervisor "$TEMP_PUB_FILE" "container-public-keys" +send_file_to_hypervisor "$PROTOCOL_FILE" "container-port-maps" +send_file_to_hypervisor "$ENV_FOLDER_PATH" "container-env-vars" +send_file_to_hypervisor "$TEMP_SERVICES_FILE_PATH" "container-services" -if [ "${REQUIRE_SERVICES^^}" == "Y" ]; then - - # Generate random (temporary) file to store install commands for needed services - RANDOM_NUM=$(shuf -i 100000-999999 -n 1) - SERVICES_FILE="services_$RANDOM_NUM.txt" - TEMP_SERVICES_FILE_PATH="/root/bin/services/$SERVICES_FILE" - touch "$TEMP_SERVICES_FILE_PATH" - - appendService - while [ "${SERVICE^^}" != "" ] || [ ! -z "$SERVICES" ]; do - if [ -z "$SERVICES" ]; then - appendService - else - if [ ! -z "$CUSTOM_SERVICES" ]; then # assumes both services and custom services passed as ENV vars - appendCustomService - else # custom services not passed as ENV var, so must prompt the user for their custom services - setUpService - while [ "${SETUP_CUSTOM_SERVICE_INSTALLATION^^}" == "Y" ]; do - appendCustomService - setUpService - done - fi - break - fi - done -fi +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +echo -e "${BOLD}${MAGENTA}🚀 Starting Container Creation...${RESET}" +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -echo -e "\n✅ Deployment Process Finished.\n" +ssh -t root@10.15.0.4 "bash -c '/var/lib/vz/snippets/clone-clxc.sh \ +'\''$CONTAINER_NAME'\'' \ +'\''$CONTAINER_PASSWORD'\'' \ +'\''$HTTP_PORT'\'' \ +'\''$PROXMOX_USERNAME'\'' \ +'\''$PUB_FILE'\'' \ +'\''$PROTOCOL_BASE_FILE'\'' \ +'\''$DEPLOY_ON_START'\'' \ +'\''$PROJECT_REPOSITORY'\'' \ +'\''$PROJECT_BRANCH'\'' \ +'\''$PROJECT_ROOT'\'' \ +'\''$INSTALL_COMMAND'\'' \ +'\''$BUILD_COMMAND'\'' \ +'\''$START_COMMAND'\'' \ +'\''$RUNTIME_LANGUAGE'\'' \ +'\''$ENV_FOLDER'\'' \ +'\''$SERVICES_FILE'\'' \ +'\''$LINUX_DISTRIBUTION'\'' \ +'\''$MULTI_COMPONENT'\'' +'\''$ROOT_START_COMMAND'\'' '" + + +rm -rf "$PROTOCOL_FILE" +rm -rf "$TEMP_PUB_FILE" +rm -rf "$TEMP_SERVICES_FILE_PATH" +rm -rf "$ENV_FOLDER_PATH" + +unset CONFIRM_PASSWORD +unset CONTAINER_PASSWORD +unset PUBLIC_KEY diff --git a/container maintenance/update-container.sh b/container maintenance/update-container.sh index 4714ac6d..39ccbde6 100644 --- a/container maintenance/update-container.sh +++ b/container maintenance/update-container.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to automatically fetch new contents from a branch, push them to container, and restart intern -# Last Modified on July 11th, 2025 by Maxwell Klema +# Last Modified on July 17th, 2025 by Maxwell Klema # ---------------------------------------- RESET="\033[0m" @@ -38,10 +38,6 @@ echo "✅ The repository link you provided, \"$PROJECT_REPOSITORY\", was valid." # Get Project Branch -if [ -z "$PROJECT_BRANCH" ]; then - read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH -fi - if [ "$PROJECT_BRANCH" == "" ]; then PROJECT_BRANCH="main" fi @@ -55,10 +51,10 @@ while [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; do done -# Get Project Root Directroy +# # Get Project Root Directroy -if [ -z "$PROJECT_ROOT" ]; then - read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT +if [ "$PROJECT_ROOT" == "" ]; then + PROJECT_ROOT="/" fi VALID_PROJECT_ROOT=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateRepo \"$PROJECT_REPOSITORY\" \"$PROJECT_BRANCH\" \"$PROJECT_ROOT\"") @@ -69,77 +65,94 @@ while [ "$VALID_PROJECT_ROOT" == "false" ]; do VALID_PROJECT_ROOT=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateRepo \"$PROJECT_REPOSITORY\" \"$PROJECT_BRANCH\" \"$PROJECT_ROOT\"") done -# Get Install Command ======== - -if [ -z "$INSTALL_COMMAND" ]; then - read -p "📦 Enter the install command (e.g., 'npm install') → " INSTALL_COMMAND -fi - -# Get Build Command ======== - -if [ -z "$BUILD_COMMAND" ]; then - read -p "🏗️ Enter the build command (leave blank if no build command) → " BUILD_COMMAND -fi - -# Get Start Command ======== +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") -if [ -z "$START_COMMAND" ]; then - read -p "🚦 Enter the start command (e.g., 'npm start', 'python app.py') → " START_COMMAND +if [ "$PROJECT_ROOT" == "" ] || [ "$PROJECT_ROOT" == "/" ]; then + PROJECT_ROOT="." fi -while [ "$START_COMMAND" == "" ]; do - echo "⚠️ The start command cannot be blank. Please try again." - read -p "🚦 Enter the start command (e.g., 'npm start') → " START_COMMAND -done - -# Get Runtime Language ======== +# Update Container with New Contents from repository -if [ -z "$RUNTIME_LANGUAGE" ]; then - read -p "🖥️ Enter the underlying runtime environment (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE -fi +startComponentPVE1() { + + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" + INSTALL_CMD="$5" + + if [ "${RUNTIME^^}" == "NODEJS" ]; then + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 > /dev/null + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD" > /dev/null + pct exec $CONTAINER_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD'" > /dev/null + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 > /dev/null + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 > /dev/null + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD" > /dev/null + pct exec $CONTAINER_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'" > /dev/null + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 > /dev/null + fi +} -while [ "${RUNTIME_LANGUAGE^^}" != "NODEJS" ] && [ "${RUNTIME_LANGUAGE^^}" != "PYTHON" ]; do - echo "⚠️ Sorry, that runtime environment is not yet supported. Only \"nodejs\" and \"python\" are currently supported." - read -p "🖥️ Enter the underlying runtime environment (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE -done +startComponentPVE2() { -REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" + INSTALL_CMD="$5" -# Update Container with New Contents from repository - -if (( "$CONTAINER_ID" % 2 == 0 )); then - if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then + if [ "${RUNTIME^^}" == "NODEJS" ]; then ssh root@10.15.0.5 " pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && - pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' - pct exec $CONTAINER_ID -- bash -c '$INSTALL_COMMAND_COMMAND' && '$BUILD_COMMAND' - pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && pm2 start bash -- -c \"$START_COMMAND\"' + pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"' > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 - " - elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then + " > /dev/null + elif [ "${RUNTIME^^}" == "PYTHON" ]; then ssh root@10.15.0.5 " pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && - pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' - pct exec $CONTAINER_ID -- bash -c '$INSTALL_COMMAND' && '$BUILD_COMMAND' - pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate && $START_COMMAND'\" /dev/null + pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 + pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\" > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 - " + " > /dev/null fi +} + +if (( "$CONTAINER_ID" % 2 == 0 )); then + startComponentPVE2 else + startComponentPVE1 +fi - if [ "${RUNTIME_LANGUAGE^^}" == "NODEJS" ]; then - pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" - pct exec $CONTAINER_ID -- bash -c "$INSTALL_COMMAND_COMMAND' && '$BUILD_COMMAND" - pct exec $CONTAINER_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && pm2 start bash -- -c \"$START_COMMAND\"" - pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 - elif [ "${RUNTIME_LANGUAGE^^}" == "PYTHON" ]; then - pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" - pct exec $CONTAINER_ID -- bash -c "$INSTALL_COMMAND' && '$BUILD_COMMAND" - pct exec $CONTAINER_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && source venv/bin/activate && $START_COMMAND'" /dev/null - pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 +if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + INSTALL=$(echo "$INSTALL_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + + if (( "$CONTAINER_ID" % 2 == 0 )); then + startComponentPVE2 "$RUNTIME" "$BUILD" "$START" "$COMPONENT" "$INSTALL" + else + startComponentPVE1 "$RUNTIME" "$BUILD" "$START" "$COMPONENT" "$INSTALL" + fi + done + if [ ! -z "$START_ON_ROOT" ]; then; + if (( "$CONTAINER_ID" % 2 == 0 )); then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + else + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" > /dev/null 2>&1 + fi fi + # startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." fi echo "✅ Container $CONTAINER_ID has been updated with new contents from branch \"$PROJECT_BRANCH\" on repository \"$PROJECT_REPOSITORY\"." diff --git a/deployment-scripts/gatherEnvVars.sh b/deployment-scripts/gatherEnvVars.sh index c577edd4..cd9b9d5a 100644 --- a/deployment-scripts/gatherEnvVars.sh +++ b/deployment-scripts/gatherEnvVars.sh @@ -35,7 +35,8 @@ if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then # generate random temp .env folder to store all env files for different components RANDOM_NUM=$(shuf -i 100000-999999 -n 1) ENV_FOLDER="env_$RANDOM_NUM" - mkdir -p "/root/bin/env/$ENV_FOLDER" + ENV_FOLDER_PATH="/root/bin/env/$ENV_FOLDER" + mkdir -p "$ENV_FOLDER_PATH" if [ "${MULTI_COMPONENT^^}" == "Y" ]; then if [ ! -z "$CONTAINER_ENV_VARS" ]; then # Environment Variables @@ -57,7 +58,7 @@ if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then gatherComponentDir "Enter the path of your component to enter environment variables" while [ "$COMPONENT_PATH" != "" ]; do - addComponent "$key" + addComponent "$COMPONENT_PATH" ENV_FILE_NAME=$(echo "$COMPONENT_PATH" | tr '/' '_') ENV_FILE="$ENV_FILE_NAME.txt" ENV_FILE_PATH="/root/bin/env/$ENV_FOLDER/$ENV_FILE" From dde80eba99312093cf33a599d09a691fbba93e70 Mon Sep 17 00:00:00 2001 From: maxklema Date: Sun, 20 Jul 2025 01:29:09 +0000 Subject: [PATCH 18/25] added runner setup script --- container creation/setup-runner.sh | 79 ++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 container creation/setup-runner.sh diff --git a/container creation/setup-runner.sh b/container creation/setup-runner.sh new file mode 100644 index 00000000..817f5f99 --- /dev/null +++ b/container creation/setup-runner.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# A script for cloning a Distro template, installing, and starting a runner on it. +# Last Modified by Maxwell Klema on July 19th, 2025 +# ------------------------------------------------ + +BOLD='\033[1m' +RESET='\033[0m' + +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +echo "🧬 Cloning a Template and installing a Runner" +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + +# Validating Container Name ===== + +set +e + +source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh #Authenticate User +source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh #Ensure container does not exist. + +CONTAINER_EXISTS=$? +if [ "$CONTAINER_EXISTS" != 1 ]; then + exit $CONTAINER_EXISTS; # Container is not free to user, either someone else owns it or the user owns it. +fi + +# Cloning Container Template and Setting it up ===== + +# Get correct template to clone and package manager +if [ ${LINUX_DISTRIBUTION^^} == "DEBIAN" ]; then + PACKAGE_MANAGER="apt" + CTID_TEMPLATE="114" +elif [ "${LINUX_DISTRIBUTION^^}" == "ROCKY" ]; then + PACKAGE_MANAGER="dnf" + CTID_TEMPLATE="113" +fi + +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") +REPO_BASE_NAME_WITH_OWNER=$(echo "$PROJECT_REPOSITORY" | cut -d'/' -f4) + +NEXT_ID=$(pvesh get /cluster/nextid) #Get the next available LXC ID + +# Create the Container Clone +echo "⏳ Cloning Container..." +pct clone $CTID_TEMPLATE $NEXT_ID \ + --hostname $CONTAINER_NAME \ + --full true > /dev/null 2>&1 + +# Set Container Options +echo "⏳ Setting Container Properties..." +pct set $NEXT_ID \ + --tags "$PROXMOX_USERNAME" \ + --onboot 1 > /dev/null 2>&1 + +pct start $NEXT_ID > /dev/null 2>&1 +pveum aclmod /vms/$NEXT_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser > /dev/null 2>&1 + +# Set password inside the container and install some pacakages +echo "📦 Updating Packages.." +pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 +pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER upgrade -y" > /dev/null > /dev/null 2>&1 +pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER install -y sudo tmux libicu perl-Digest-SHA git curl vim tar" > /dev/null 2>&1 + +# Setting Up Github Runner ===== + +# Get Temporary Token +echo "🪙 Getting Authentication Token..." +AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/registration-token --header "Authorization: token $PAT") +TOKEN=$(echo "$AUTH_TOKEN_RESPONSE" | jq -r '.token') + +pct enter $NEXT_ID < /dev/null +mkdir actions-runner && cd actions-runner && \ +curl -o actions-runner-linux-x64-2.326.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.326.0/actions-runner-linux-x64-2.326.0.tar.gz && \ +echo "9c74af9b4352bbc99aecc7353b47bcdfcd1b2a0f6d15af54a99f54a0c14a1de8 actions-runner-linux-x64-2.326.0.tar.gz" | shasum -a 256 -c && \ +tar xzf ./actions-runner-linux-x64-2.326.0.tar.gz && \ +export RUNNER_ALLOW_RUNASROOT=1 && \ +./config.sh --url $PROJECT_REPOSITORY --token $TOKEN --labels $CONTAINER_NAME +EOF + +# Start Runner +pct exec $NEXT_ID -- bash -c "tmux new-session -d 'cd /actions-runner && export RUNNER_ALLOW_RUNASROOT=1 && ./run.sh'" > /dev/null 2>&1 \ No newline at end of file From 59a2784fd95b044a459635858f4c0d7f8a5ef1e7 Mon Sep 17 00:00:00 2001 From: maxklema Date: Tue, 22 Jul 2025 19:25:41 +0000 Subject: [PATCH 19/25] CI/CD changes for self-hosted runners --- container creation/create-container.sh | 252 ++++++++++-------- container creation/setup-runner.sh | 55 +++- .../check-container-exists.sh | 28 +- container maintenance/delete-container.sh | 34 +-- container maintenance/finish-migration.sh | 87 ++++++ .../helper-scripts/delete-runner.sh | 46 ++++ .../helper-scripts/repository_status.sh | 37 +++ .../verify_container_ownership.sh | 23 +- container maintenance/update-container.sh | 59 ++-- 9 files changed, 455 insertions(+), 166 deletions(-) create mode 100644 container maintenance/finish-migration.sh create mode 100644 container maintenance/helper-scripts/delete-runner.sh create mode 100644 container maintenance/helper-scripts/repository_status.sh diff --git a/container creation/create-container.sh b/container creation/create-container.sh index b0a3865e..22588d1d 100644 --- a/container creation/create-container.sh +++ b/container creation/create-container.sh @@ -1,44 +1,8 @@ #!/bin/bash # Script to create the pct container, run register container, and migrate container accordingly. -# Last Modified by July 11th, 2025 by Maxwell Klema - -trap cleanup SIGINT SIGTERM SIGHUP - -CONTAINER_NAME="$1" -CONTAINER_PASSWORD="$2" -HTTP_PORT="$3" -PROXMOX_USERNAME="$4" -PUB_FILE="$5" -PROTOCOL_FILE="$6" - -# Deployment ENVS -DEPLOY_ON_START="$7" -PROJECT_REPOSITORY="$8" -PROJECT_BRANCH="$9" -PROJECT_ROOT="${10}" -INSTALL_COMMAND="${11}" -BUILD_COMMAND="${12}" -START_COMMAND="${13}" -RUNTIME_LANGUAGE="${14}" -ENV_BASE_FOLDER="${15}" -SERVICES_BASE_FILE="${16}" -LINUX_DISTRO="${17}" -MULTI_COMPONENTS="${18}" -ROOT_START_COMMAND="${19}" - -if [ ${LINUX_DISTRO^^} == "DEBIAN" ]; then - PACKAGE_MANAGER="apt" - CTID_TEMPLATE="114" -elif [ "${LINUX_DISTRO^^}" == "ROCKY" ]; then - PACKAGE_MANAGER="dnf" - CTID_TEMPLATE="113" -fi - -REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") -NEXT_ID=$(pvesh get /cluster/nextid) #Get the next available LXC ID +# Last Modified by July 22nd, 2025 by Maxwell Klema # Run cleanup commands in case script is interrupted - function cleanup() { BOLD='\033[1m' @@ -64,44 +28,102 @@ function cleanup() exit 1 } +# Echo Container Details +function echoContainerDetails() { + BOLD='\033[1m' + BLUE='\033[34m' + MAGENTA='\033[35m' + GREEN='\033[32m' + RESET='\033[0m' -# Create the Container Clone + echo -e "📦 ${BLUE}Container ID :${RESET} $CONTAINER_ID" + echo -e "🌐 ${MAGENTA}Internal IP :${RESET} $CONTAINER_IP" + echo -e "🔗 ${GREEN}Domain Name :${RESET} https://$CONTAINER_NAME.opensource.mieweb.org" + echo -e "🛠️ ${BLUE}SSH Access :${RESET} ssh -p $SSH_PORT root@$CONTAINER_NAME.opensource.mieweb.org" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +} -echo "⏳ Cloning Container..." -pct clone $CTID_TEMPLATE $NEXT_ID \ - --hostname $CONTAINER_NAME \ - --full true > /dev/null 2>&1 +trap cleanup SIGINT SIGTERM SIGHUP -# Set Container Options +CONTAINER_NAME="${CONTAINER_NAME,,}" -echo "⏳ Setting Container Properties..." -pct set $NEXT_ID \ - --tags "$PROXMOX_USERNAME" \ - --onboot 1 > /dev/null 2>&1 +CONTAINER_NAME="$1" +CONTAINER_PASSWORD="$2" +GH_ACTION="$3" +HTTP_PORT="$4" +PROXMOX_USERNAME="$5" +PUB_FILE="$6" +PROTOCOL_FILE="$7" -pct start $NEXT_ID > /dev/null 2>&1 -pveum aclmod /vms/$NEXT_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser > /dev/null 2>&1 -#pct delete $NEXT_ID +# Deployment ENVS +DEPLOY_ON_START="$8" +PROJECT_REPOSITORY="$9" +PROJECT_BRANCH="${10}" +PROJECT_ROOT="${11}" +INSTALL_COMMAND="${12}" +BUILD_COMMAND="${13}" +START_COMMAND="${14}" +RUNTIME_LANGUAGE="${15}" +ENV_BASE_FOLDER="${16}" +SERVICES_BASE_FILE="${17}" +LINUX_DISTRO="${18}" +MULTI_COMPONENTS="${19}" +ROOT_START_COMMAND="${20}" -# Get the Container IP Address and install some packages +if [ ${LINUX_DISTRO^^} == "DEBIAN" ]; then + PACKAGE_MANAGER="apt-get" + CTID_TEMPLATE="114" +elif [ "${LINUX_DISTRO^^}" == "ROCKY" ]; then + PACKAGE_MANAGER="dnf" + CTID_TEMPLATE="113" +fi -echo "⏳ Waiting for DHCP to allocate IP address to container..." -sleep 10 +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") -echo "⏳ Updating Packages.." +if [ "${GH_ACTION^^}" != "Y" ]; then + # Create the Container Clone -CONTAINER_IP=$(pct exec $NEXT_ID -- hostname -I | awk '{print $1}') -pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER upgrade -y" > /dev/null -pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER install -y sudo git curl vim" > /dev/null -if [ -f "/var/lib/vz/snippets/container-public-keys/$PUB_FILE" ]; then - pct exec $NEXT_ID -- touch ~/.ssh/authorized_keys > /dev/null 2>&1 - pct exec $NEXT_ID -- bash -c "cat > ~/.ssh/authorized_keys"< /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 - rm -rf /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 -fi + CONTAINER_ID=$(pvesh get /cluster/nextid) #Get the next available LXC ID + + echo "⏳ Cloning Container..." + pct clone $CTID_TEMPLATE $CONTAINER_ID \ + --hostname $CONTAINER_NAME \ + --full true > /dev/null 2>&1 + + # Set Container Options + + echo "⏳ Setting Container Properties..." + pct set $CONTAINER_ID \ + --tags "$PROXMOX_USERNAME" \ + --onboot 1 > /dev/null 2>&1 + + pct start $CONTAINER_ID > /dev/null 2>&1 + pveum aclmod /vms/$CONTAINER_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser > /dev/null 2>&1 + #pct delete $CONTAINER_ID -# Set password inside the container + # Get the Container IP Address and install some packages -pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 + echo "⏳ Waiting for DHCP to allocate IP address to container..." + sleep 10 + + echo "⏳ Updating Packages.." + + pct exec $CONTAINER_ID -- bash -c "$PACKAGE_MANAGER upgrade -y" > /dev/null + pct exec $CONTAINER_ID -- bash -c "$PACKAGE_MANAGER install -y sudo git curl vim" > /dev/null + if [ -f "/var/lib/vz/snippets/container-public-keys/$PUB_FILE" ]; then + pct exec $CONTAINER_ID -- touch ~/.ssh/authorized_keys > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "cat > ~/.ssh/authorized_keys"< /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 + rm -rf /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 + fi + + # Set password inside the container + + pct exec $CONTAINER_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 +else + CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$CONTAINER_NAME" '$3 == name {print $1}') +fi + +CONTAINER_IP=$(pct exec $CONTAINER_ID -- hostname -I | awk '{print $1}') # Attempt to Automatically Deploy Project Inside Container @@ -120,10 +142,10 @@ fi # Run Contianer Provision Script to add container to port_map.json if [ -f "/var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE" ]; then - /var/lib/vz/snippets/register-container.sh $NEXT_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE + /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE rm -rf /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE > /dev/null 2>&1 else - /var/lib/vz/snippets/register-container.sh $NEXT_ID $HTTP_PORT + /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT fi SSH_PORT=$(iptables -t nat -S PREROUTING | grep "to-destination $CONTAINER_IP:22" | awk -F'--dport ' '{print $2}' | awk '{print $1}' | head -n 1 || true) @@ -139,57 +161,73 @@ startProject() { if [ "${RUNTIME^^}" == "NODEJS" ]; then if [ "$BUILD_CMD" == "" ]; then - ssh root@10.15.0.5 "pct exec $NEXT_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 else - ssh root@10.15.0.5 "pct exec $NEXT_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && $BUILD_CMD && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && $BUILD_CMD && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 fi elif [ "${RUNTIME^^}" == "PYTHON" ]; then if [ "$BUILD_CMD" == "" ]; then - ssh root@10.15.0.5 "pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\"" > /dev/null 2>&1 + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\"" > /dev/null 2>&1 else - ssh root@10.15.0.5 "pct exec $NEXT_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 fi fi } -if (( $NEXT_ID % 2 == 0 )); then - pct stop $NEXT_ID > /dev/null 2>&1 - pct migrate $NEXT_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 - ssh root@10.15.0.5 "pct start $NEXT_ID" > /dev/null 2>&1 - ssh root@10.15.0.5 "pct exec $NEXT_ID -- bash -c 'chmod 700 ~/.bashrc'" > /dev/null 2>&1 # enable full R/W/X permissions - ssh root@10.15.0.5 "pct set $NEXT_ID --memory 4096 --swap 0 --cores 4" > /dev/null 2>&1 - if [ "${DEPLOY_ON_START^^}" == "Y" ]; then - if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then - for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do - START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') - BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - if [ "$BUILD" == "null" ]; then - BUILD="" - fi - startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" - done - if [ ! -z "$ROOT_START_COMMAND" ]; then - ssh root@10.15.0.5 "pct exec $NEXT_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 - fi - else - startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." - fi - fi - ssh root@10.15.0.5 "pct set $NEXT_ID --memory 2048 --swap 0 --cores 2" +if (( $CONTAINER_ID % 2 == 0 )); then + if [ "${GH_ACTION^^}" != "Y" ]; then + pct stop $CONTAINER_ID > /dev/null 2>&1 + pct migrate $CONTAINER_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 + ssh root@10.15.0.5 "pct start $CONTAINER_ID" > /dev/null 2>&1 + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'chmod 700 ~/.bashrc'" > /dev/null 2>&1 # enable full R/W/X permissions + ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4" > /dev/null 2>&1 + if [ "${DEPLOY_ON_START^^}" == "Y" ]; then + if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" + done + if [ ! -z "$ROOT_START_COMMAND" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi + else + startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + fi + fi + ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2" + else + echoContainerDetails + echo "NOTE: Your Container needs to Migrate. Wait ~2 minutes before trying to SSH or navigate to the URL" + + CMD=( + bash /var/lib/vz/snippets/finish-migration.sh + "$CONTAINER_ID" + "$CONTAINER_NAME" + "$REPO_BASE_NAME" + "$REPO_BASE_NAME_WITH_OWNER" + "$SSH_PORT" + "$CONTAINER_IP" + "$PROJECT_ROOT" + "$ROOT_START_COMMAND" + "$DEPLOY_ON_START" + "$MULTI_COMPONENTS" + "$START_COMMAND" + "$BUILD_COMMAND" + "$RUNTIME_LANGUAGE" + ) + + # Safely quote each argument for the shell + QUOTED_CMD=$(printf ' %q' "${CMD[@]}") + + tmux new-session -d -s finish_migration "$QUOTED_CMD" + fi fi -# Echo Container Details -BOLD='\033[1m' -BLUE='\033[34m' -MAGENTA='\033[35m' -GREEN='\033[32m' -RESET='\033[0m' - -echo -e "📦 ${BLUE}Container ID :${RESET} $NEXT_ID" -echo -e "🌐 ${MAGENTA}Internal IP :${RESET} $CONTAINER_IP" -echo -e "🔗 ${GREEN}Domain Name :${RESET} https://$CONTAINER_NAME.opensource.mieweb.org" -echo -e "🛠️ ${BLUE}SSH Access :${RESET} ssh -p $SSH_PORT root@$CONTAINER_NAME.opensource.mieweb.org" -echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +echoContainerDetails diff --git a/container creation/setup-runner.sh b/container creation/setup-runner.sh index 817f5f99..33261665 100644 --- a/container creation/setup-runner.sh +++ b/container creation/setup-runner.sh @@ -1,6 +1,6 @@ #!/bin/bash # A script for cloning a Distro template, installing, and starting a runner on it. -# Last Modified by Maxwell Klema on July 19th, 2025 +# Last Modified by Maxwell Klema on July 20th, 2025 # ------------------------------------------------ BOLD='\033[1m' @@ -18,6 +18,7 @@ source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh #Authentic source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh #Ensure container does not exist. CONTAINER_EXISTS=$? + if [ "$CONTAINER_EXISTS" != 1 ]; then exit $CONTAINER_EXISTS; # Container is not free to user, either someone else owns it or the user owns it. fi @@ -53,17 +54,25 @@ pct set $NEXT_ID \ pct start $NEXT_ID > /dev/null 2>&1 pveum aclmod /vms/$NEXT_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser > /dev/null 2>&1 +sleep 5 +echo "⏳ DHCP Allocating IP Address..." +CONTAINER_IP=$(pct exec $NEXT_ID -- hostname -I | awk '{print $1}') + # Set password inside the container and install some pacakages -echo "📦 Updating Packages.." +echo "📦 Updating Packages..." pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER upgrade -y" > /dev/null > /dev/null 2>&1 -pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER install -y sudo tmux libicu perl-Digest-SHA git curl vim tar" > /dev/null 2>&1 +if [ "${LINUX_DISTRIBUTION^^}" == "DEBIAN" ]; then + pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER install -y sudo git curl vim tar tmux sshpass jq" > /dev/null 2>&1 +elif [ "${LINUX_DISTRIBUTION^^}" == "ROCKY" ]; then + pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER install -y sudo git curl vim tar tmux libicu perl-Digest-SHA sshpass jq" > /dev/null 2>&1 +fi # Setting Up Github Runner ===== # Get Temporary Token echo "🪙 Getting Authentication Token..." -AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/registration-token --header "Authorization: token $PAT") +AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/registration-token --header "Authorization: token $GITHUB_PAT") TOKEN=$(echo "$AUTH_TOKEN_RESPONSE" | jq -r '.token') pct enter $NEXT_ID < /dev/null @@ -75,5 +84,39 @@ export RUNNER_ALLOW_RUNASROOT=1 && \ ./config.sh --url $PROJECT_REPOSITORY --token $TOKEN --labels $CONTAINER_NAME EOF -# Start Runner -pct exec $NEXT_ID -- bash -c "tmux new-session -d 'cd /actions-runner && export RUNNER_ALLOW_RUNASROOT=1 && ./run.sh'" > /dev/null 2>&1 \ No newline at end of file +# Generate RSA Keys ===== + +echo "🔑 Generating RSA Key Pair..." +pct exec $NEXT_ID -- bash -c "ssh-keygen -t rsa -N '' -f /root/.ssh/id_rsa -q" +PUB_KEY=$(pct exec $NEXT_ID -- bash -c "cat /root/.ssh/id_rsa.pub") + +# Place public key in all necessary authorized_keys files +echo "$PUB_KEY" >> /home/create-container/.ssh/authorized_keys +echo "$PUB_KEY" >> /home/update-container/.ssh/authorized_keys +echo "$PUB_KEY" >> /home/delete-container/.ssh/authorized_keys +echo "$PUB_KEY" >> /home/container-exists/.ssh/authorized_keys + +ssh root@10.15.234.122 "echo \"$PUB_KEY\" >> /root/.ssh/authorized_keys" + +echo "🔑 Creating Service File..." +pct exec $NEXT_ID -- bash -c "cat < /etc/systemd/system/github-runner.service +[Unit] +Description=GitHub Actions Runner +After=network.target + +[Service] +Type=simple +WorkingDirectory=/actions-runner +Environment=\"RUNNER_ALLOW_RUNASROOT=1\" +ExecStart=/actions-runner/run.sh +Restart=always + +[Install] +WantedBy=multi-user.target +EOF" + +pct exec $NEXT_ID -- systemctl daemon-reload +pct exec $NEXT_ID -- systemctl enable github-runner +pct exec $NEXT_ID -- systemctl start github-runner + +exit 3 diff --git a/container maintenance/check-container-exists.sh b/container maintenance/check-container-exists.sh index 669c79df..d8ca9710 100644 --- a/container maintenance/check-container-exists.sh +++ b/container maintenance/check-container-exists.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to check if a container exists -# Last Modified by Maxwell Klema on July 13th, 2025 +# Last Modified by Maxwell Klema on July 22nd, 2025 # ----------------------------------------------------- RESET="\033[0m" @@ -11,7 +11,31 @@ echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━ echo -e "${BOLD}${MAGENTA}🔎 Check Container Exists ${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +set +e +TYPE_RUNNER="true" source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh -exit 0 # Container Exists and is owned by the user \ No newline at end of file +STATUS=$? + +if [ "$STATUS" != 0 ]; then + exit 1; +fi + +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") + +# Check if repository folder is present. + +if [ "$PVE1" == "true" ]; then + if pct exec $CONTAINER_ID -- test -d /root/$REPO_BASE_NAME; then + exit 2; # Update Repository + else + exit 0; # Clone Repository + fi +else + if ssh 10.15.0.5 "pct exec $CONTAINER_ID -- test -d /root/$REPO_BASE_NAME"; then + exit 2; # Update Repository + else + exit 0; # Clone Repository + fi +fi \ No newline at end of file diff --git a/container maintenance/delete-container.sh b/container maintenance/delete-container.sh index 63cc1b29..c3538ce3 100644 --- a/container maintenance/delete-container.sh +++ b/container maintenance/delete-container.sh @@ -11,29 +11,19 @@ echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━ echo -e "${BOLD}${MAGENTA}🗑️ Delete Container ${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh -source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh +CMD=( +bash /var/lib/vz/snippets/helper-scripts/delete-runner.sh +"$PROJECT_REPOSITORY" +"$GITHUB_PAT" +"$PROXMOX_USERNAME" +"$PROXMOX_PASSWORD" +"$CONTAINER_NAME" +) -# Delete Container +# Safely quote each argument for the shell +QUOTED_CMD=$(printf ' %q' "${CMD[@]}") -echo "🔄 Deleting container with name \"$CONTAINER_NAME\"..." +tmux new-session -d -s delete-runner "$QUOTED_CMD" -if (( $CONTAINER_ID % 2 == 0 )); then - if ssh root@10.15.0.5 "pct status $CONTAINER_ID" | grep -q "status: running"; then - ssh root@10.15.0.5 "pct stop $CONTAINER_ID && pct destroy $CONTAINER_ID" > /dev/null 2>&1 - else - ssh root@10.15.0.5 "pct destroy $CONTAINER_ID" > /dev/null 2>&1 - fi -else - if pct status "$CONTAINER_ID" | grep -q "status: running"; then - pct stop "$CONTAINER_ID" && pct destroy "$CONTAINER_ID" > /dev/null 2>&1 - else - pct destroy "$CONTAINER_ID" > /dev/null 2>&1 - fi -fi - -echo "🧹 Running Cleanup Tasks..." -source /usr/local/bin/prune_iptables.sh - -echo "✅ Container with name \"$CONTAINER_NAME\" has been permanently deleted." +echo "✅ Container with name \"$CONTAINER_NAME\" will been permanently deleted." exit 0 # Container Deleted Successfully \ No newline at end of file diff --git a/container maintenance/finish-migration.sh b/container maintenance/finish-migration.sh new file mode 100644 index 00000000..0969da8e --- /dev/null +++ b/container maintenance/finish-migration.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Script ran by a virtual terminal session to migrate a container +# Script is only ran on GH action workflows when runner disconnects due to migrations +# Last Modified by Maxwell Klema on July 22nd, 2025 +# ------------------------------------------------ + +CONTAINER_ID="$1" +CONTAINER_NAME="$2" +REPO_BASE_NAME="$3" +REPO_BASE_NAME_WITH_OWNER="$4" +SSH_PORT="$5" +CONTAINER_IP="$6" +PROJECT_ROOT="$7" +ROOT_START_COMMAND="$8" +DEPLOY_ON_START="$9" +MULTI_COMPONENT="${10}" +START_COMMAND="${11}" +BUILD_COMMAND="${12}" +RUNTIME_LANGUAGE="${13}" + +echo "$1 $2 $3 $4 $5 $6 $7 $8" +echo "Deploy on start: $9" >> "log.txt" +echo "Multi Component ${10}" >> "log.txt" +echo "Start command: ${11}" >> "log.txt" +echo "Build command: ${12}" >> "log.txt" +echo "Runtime language: ${13}" >> "log.txt" + +CONTAINER_NAME="${CONTAINER_NAME,,}" + +sleep 10 + +pct stop $CONTAINER_ID > /dev/null 2>&1 +pct migrate $CONTAINER_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 + +sleep 40 # wait for migration to finish (fix this later) + +ssh root@10.15.0.5 "pct start $CONTAINER_ID" +ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'chmod 700 ~/.bashrc'" # enable full R/W/X permissions +ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4" +ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- systemctl start github-runner" +ssh root@10.15.0.5 "pct start $CONTAINER_ID" > /dev/null 2>&1 +ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'chmod 700 ~/.bashrc'" > /dev/null 2>&1 # enable full R/W/X permissions +ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4" > /dev/null 2>&1 + +startProject() { + + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" + + if [ "${RUNTIME^^}" == "NODEJS" ]; then + if [ "$BUILD_CMD" == "" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 + else + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && $BUILD_CMD && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 + fi + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + if [ "$BUILD_CMD" == "" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\"" > /dev/null 2>&1 + else + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 + fi + fi + +} + +if [ "${DEPLOY_ON_START^^}" == "Y" ]; then + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" + done + if [ ! -z "$ROOT_START_COMMAND" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi + else + startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + fi +fi + +ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2" \ No newline at end of file diff --git a/container maintenance/helper-scripts/delete-runner.sh b/container maintenance/helper-scripts/delete-runner.sh new file mode 100644 index 00000000..8fdbf997 --- /dev/null +++ b/container maintenance/helper-scripts/delete-runner.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Remove Github Runner Associated with Container (and Container itself too) +# Last Modified by Maxwell Klema on July 22nd, 2025 +# ------------------------------------------------ + +PROJECT_REPOSITORY="$1" +GITHUB_PAT="$2" +PROXMOX_USERNAME="$3" +PROXMOX_PASSWORD="$4" +CONTAINER_NAME="$5" + +sleep 2 + +# Delete Container + +CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$CONTAINER_NAME" '$3 == name {print $1}') + +if (( $CONTAINER_ID % 2 == 0 )); then + if ssh root@10.15.0.5 "pct status $CONTAINER_ID" | grep -q "status: running"; then + ssh root@10.15.0.5 "pct stop $CONTAINER_ID && pct destroy $CONTAINER_ID" > /dev/null 2>&1 + else + ssh root@10.15.0.5 "pct destroy $CONTAINER_ID" > /dev/null 2>&1 + fi +else + if pct status "$CONTAINER_ID" | grep -q "status: running"; then + pct stop "$CONTAINER_ID" && pct destroy "$CONTAINER_ID" > /dev/null 2>&1 + else + pct destroy "$CONTAINER_ID" > /dev/null 2>&1 + fi +fi + +source /usr/local/bin/prune_iptables.sh + +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") +REPO_BASE_NAME_WITH_OWNER=$(echo "$PROJECT_REPOSITORY" | cut -d'/' -f4) + +RUNNERS=$(curl --location https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners --header "Authorization: token $GITHUB_PAT") + +while read -r RUNNER; do + RUNNER_NAME=$(echo "$RUNNER" | jq -r '.name') + if [ "$RUNNER_NAME" == "$CONTAINER_NAME" ]; then + RUNNER_ID=$(echo "$RUNNER" | jq -r '.id') + curl --location --request DELETE "https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/$RUNNER_ID" \ + --header "Authorization: token $GITHUB_PAT" + fi +done < <(echo "$RUNNERS" | jq -c '.runners[]') \ No newline at end of file diff --git a/container maintenance/helper-scripts/repository_status.sh b/container maintenance/helper-scripts/repository_status.sh new file mode 100644 index 00000000..5ec73b76 --- /dev/null +++ b/container maintenance/helper-scripts/repository_status.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Helper script to determine if container needs to clone repository or simply update it +# Last Modified by Maxwell Klema on July 21st, 2025 +# ------------------------------------------------- + +set +e +TYPE_RUNNER="true" +source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh +source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh + +STATUS=$? + +if [ "$STATUS" != 0 ]; then + exit 1; +fi + +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") + +# Check if repository folder is present. + +if [ "$PVE1" == "true" ]; then + if pct exec $CONTAINER_ID -- test -d /root/$REPO_BASE_NAME; then + echo "Update" + exit 2; # Update Repository + else + echo "Clone" + exit 0; # Clone Repository + fi +else + if ssh 10.15.0.5 "pct exec $CONTAINER_ID -- test -d /root/$REPO_BASE_NAME"; then + echo "Update" + exit 2; # Update Repository + else + echo "Clone" + exit 0; # Clone Repository + fi +fi \ No newline at end of file diff --git a/container maintenance/helper-scripts/verify_container_ownership.sh b/container maintenance/helper-scripts/verify_container_ownership.sh index 19ca0541..53b9e1cd 100644 --- a/container maintenance/helper-scripts/verify_container_ownership.sh +++ b/container maintenance/helper-scripts/verify_container_ownership.sh @@ -3,6 +3,8 @@ # Last Modified by Maxwell Klema on July 13th, 2025 # ----------------------------------------------------- +CONTAINER_NAME="${CONTAINER_NAME,,}" + if [ -z "$CONTAINER_NAME" ]; then read -p "Enter Container Name → " CONTAINER_NAME fi @@ -10,17 +12,26 @@ fi CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$CONTAINER_NAME" '$3 == name {print $1}') if [ -z "$CONTAINER_ID" ]; then - echo "❌ Container with name \"$CONTAINER_NAME\" does not exist." - exit 1 + echo "✅ Container with name \"$CONTAINER_NAME\" is available for use." + return 1 fi -if (( $CONTAINER_ID % 2 == 0 )); then - CONTAINER_OWNERSHIP=$(ssh root@10.15.0.5 "pct config \"$CONTAINER_ID\" | grep "tags" | grep \"$PROXMOX_USERNAME\"") +if [ "$TYPE_RUNNER" != "true" ]; then + if (( $CONTAINER_ID % 2 == 0 )); then + CONTAINER_OWNERSHIP=$(ssh root@10.15.0.5 "pct config \"$CONTAINER_ID\" | grep "tags" | grep \"$PROXMOX_USERNAME\"") + else + CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep -x "tags: $PROXMOX_USERNAME") + fi else - CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep -x "tags: $PROXMOX_USERNAME") + CONTAINER_OWNERSHIP=$(ssh root@10.15.0.5 "pct config \"$CONTAINER_ID\" | grep "tags" | grep \"$PROXMOX_USERNAME\"") + PVE1="false" + if [ -z "$CONTAINER_OWNERSHIP" ]; then + CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep -x "tags: $PROXMOX_USERNAME") + PVE1="true" + fi fi if [ -z "$CONTAINER_OWNERSHIP" ]; then echo "❌ You do not own the container with name \"$CONTAINER_NAME\"." - exit 2 + return 2 fi diff --git a/container maintenance/update-container.sh b/container maintenance/update-container.sh index 39ccbde6..aa587318 100644 --- a/container maintenance/update-container.sh +++ b/container maintenance/update-container.sh @@ -11,11 +11,18 @@ echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━ echo -e "${BOLD}${MAGENTA}🔄 Update Container Contents ${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +if [ "${DEPLOY_ON_START^^}" != "Y" ]; then + echo "Skipping container update because there is nothing to update." + exit 0 +fi + source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh # Get Project Details +CONTAINER_NAME="${CONTAINER_NAME,,}" + if [ -z "$PROJECT_REPOSITORY" ]; then read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY fi @@ -23,9 +30,10 @@ fi CheckRepository() { PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY#*github.com/} PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY_SHORTENED%.git} - REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED) + REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$RROJECT_REPOSITORY) } + CheckRepository while [ "$REPOSITORY_EXISTS" != "200" ]; do @@ -44,10 +52,14 @@ fi REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED/branches/$PROJECT_BRANCH) +REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY/tree/$PROJECT_BRANCH) while [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; do echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH - REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED/branches/$PROJECT_BRANCH) + if [ "PROJECT_BRANCH" == "" ]; then + PROJECT_BRANCH="main" + fi + REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY_SHORTENED/tree/$PROJECT_BRANCH) done @@ -82,17 +94,17 @@ startComponentPVE1() { INSTALL_CMD="$5" if [ "${RUNTIME^^}" == "NODEJS" ]; then - pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 > /dev/null - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD" > /dev/null - pct exec $CONTAINER_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD'" > /dev/null - pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 > /dev/null + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD'" > /dev/null 2>&1 + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 elif [ "${RUNTIME^^}" == "PYTHON" ]; then - pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 > /dev/null - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD" > /dev/null - pct exec $CONTAINER_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'" > /dev/null - pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 > /dev/null + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $INSTALL_CMD' && '$BUILD_CMD" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'" > /dev/null 2>&1 + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 fi } @@ -111,23 +123,18 @@ startComponentPVE2() { pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"' > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 - " > /dev/null + " elif [ "${RUNTIME^^}" == "PYTHON" ]; then ssh root@10.15.0.5 " pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' > /dev/null 2>&1 - pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\" > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 - " > /dev/null + " fi } -if (( "$CONTAINER_ID" % 2 == 0 )); then - startComponentPVE2 -else - startComponentPVE1 -fi if [ "${MULTI_COMPONENT^^}" == "Y" ]; then for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do @@ -145,14 +152,20 @@ if [ "${MULTI_COMPONENT^^}" == "Y" ]; then startComponentPVE1 "$RUNTIME" "$BUILD" "$START" "$COMPONENT" "$INSTALL" fi done - if [ ! -z "$START_ON_ROOT" ]; then; + if [ ! -z "$ROOT_START_COMMAND" ]; then if (( "$CONTAINER_ID" % 2 == 0 )); then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" else - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" fi fi # startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." +else + if (( "$CONTAINER_ID" % 2 == 0 )); then + startComponentPVE2 "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + else + startComponentPVE1 "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + fi fi echo "✅ Container $CONTAINER_ID has been updated with new contents from branch \"$PROJECT_BRANCH\" on repository \"$PROJECT_REPOSITORY\"." From f948df366ce30c311559c40dbb4340bbb3dc9146 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 23 Jul 2025 19:09:58 -0400 Subject: [PATCH 20/25] refactoring and template generation --- container creation/automaticLXCDeployment.sh | 158 ------------- container creation/create-container.sh | 218 +++++++----------- container creation/deployOnStart.sh | 75 ++++++ container maintenance/finish-migration.sh | 87 ------- .../helper-scripts/create-template.sh | 37 +++ container maintenance/start_services.sh | 111 +++++++++ 6 files changed, 310 insertions(+), 376 deletions(-) delete mode 100644 container creation/automaticLXCDeployment.sh create mode 100755 container creation/deployOnStart.sh delete mode 100644 container maintenance/finish-migration.sh create mode 100755 container maintenance/helper-scripts/create-template.sh create mode 100644 container maintenance/start_services.sh diff --git a/container creation/automaticLXCDeployment.sh b/container creation/automaticLXCDeployment.sh deleted file mode 100644 index 4829902b..00000000 --- a/container creation/automaticLXCDeployment.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash -# Automation Script for attempting to automatically deploy projects and services on a container -# Last Modifided by Maxwell Klema on July 16th, 2025 -# ----------------------------------------------------- - -echo "🚀 Attempting Automatic Deployment" - -REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") - -# Clone github repository from correct branch ==== - -pct enter $NEXT_ID < .env -EOF - done -else - ENV_FOLDER_BASE_NAME=$(basename "$ENV_BASE_FOLDER") - ENV_VARS=$(cat $ENV_BASE_FOLDER/$ENV_FOLDER_BASE_NAME.txt) - pct enter $NEXT_ID < .env -EOF -fi - -# Run Installation Commands ==== - -runInstallCommands() { - - RUNTIME="$1" - COMP_DIR="$2" - - if [ "${RUNTIME^^}" == "NODEJS" ]; then - if [ ${LINUX_DISTRO^^} == "DEBIAN" ]; then - pct enter $NEXT_ID < /dev/null -sudo $PACKAGE_MANAGER install -y curl -curl -fsSL https://deb.nodesource.com/setup_20.x | sudo bash - -sudo $PACKAGE_MANAGER update -y && \ -sudo $PACKAGE_MANAGER install -y nodejs && \ -npm install -g pm2 && \ -cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && sudo $INSTALL_CMD -EOF - elif [ "${LINUX_DISTRO^^}" == "ROCKY" ]; then - pct enter $NEXT_ID < /dev/null -sudo $PACKAGE_MANAGER install -y curl -curl -fsSL https://rpm.nodesource.com/setup_20.x | sudo bash - -sudo $PACKAGE_MANAGER update -y && \ -sudo $PACKAGE_MANAGER install -y nodejs && \ -npm install -g pm2 && \ -cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && sudo $INSTALL_CMD -EOF - fi - elif [ "${RUNTIME^^}" == "PYTHON" ]; then - if [ ${LINUX_DISTRO^^} == "DEBIAN" ]; then - pct enter $NEXT_ID < /dev/null -sudo apt install -y python3-venv python3-pip tmux && \ -cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && \ -python3 -m venv venv && source venv/bin/activate && \ -pip install --upgrade pip && \ -$INSTALL_CMD -EOF - elif [ "${LINUX_DISTRO^^}" == "ROCKY" ]; then - pct enter $NEXT_ID < /dev/null -sudo $PACKAGE_MANAGER install -y python python3-pip tmux && \ -cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && \ -python3 -m venv venv && source venv/bin/activate && \ -pip install --upgrade pip && \ -$INSTALL_CMD -EOF - fi - fi -} - -if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then - for COMPONENT in $(echo "$RUNTIME_LANGUAGE" | jq -r 'keys[]'); do - RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') #get runtime env - INSTALL_CMD=$(echo "$INSTALL_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') #get install command - if [ "$INSTALL_CMD" != "null" ]; then - runInstallCommands "$RUNTIME" "$COMPONENT" - fi - done -else - INSTALL_CMD=$INSTALL_COMMAND - runInstallCommands "$RUNTIME_LANGUAGE" "." -fi - -# Install Services ==== - -if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then - while read line; do - pct exec $NEXT_ID -- bash -c "$line" > /dev/null 2>&1 - done < "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" -fi - -# Build and Start Processes ==== - -startComponent() { - - RUNTIME="$1" - BUILD_CMD="$2" - START_CMD="$3" - COMP_DIR="$4" - - if [ "${RUNTIME^^}" == "NODEJS" ]; then - if [ "$BUILD_CMD" == "" ]; then - pct exec $NEXT_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD'" > /dev/null 2>&1 - else - pct enter $NEXT_ID < /dev/null -export PATH=\$PATH:/usr/local/bin && \ -$BUILD_CMD && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD' -EOF - fi - elif [ "${RUNTIME^^}" == "PYTHON" ]; then - if [ "$BUILD_CMD" == "" ]; then - pct exec $NEXT_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'" > /dev/null 2>&1 - else - pct exec $NEXT_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'" > /dev/null 2>&1 - fi - fi -} - -if (( $NEXT_ID % 2 == 1 )); then - pct set $NEXT_ID --memory 4096 --swap 0 --cores 4 > /dev/null #temporarily bump up container resources for computation hungry processes (e.g. meteor) - if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then - for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do - START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') - BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - if [ "$BUILD" == "null" ]; then - BUILD="" - fi - - startComponent "$RUNTIME" "$BUILD" "$START" "$COMPONENT" - done - if [ ! -z "$ROOT_START_COMMAND" ]; then - pct exec $NEXT_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" > /dev/null 2>&1 - fi - else - startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." - fi - pct set $NEXT_ID --memory 2048 --swap 0 --cores 2 > /dev/null -fi diff --git a/container creation/create-container.sh b/container creation/create-container.sh index 22588d1d..a4d47423 100644 --- a/container creation/create-container.sh +++ b/container creation/create-container.sh @@ -1,46 +1,46 @@ #!/bin/bash # Script to create the pct container, run register container, and migrate container accordingly. -# Last Modified by July 22nd, 2025 by Maxwell Klema +# Last Modified by July 23rd, 2025 by Maxwell Klema + +BOLD='\033[1m' +BLUE='\033[34m' +MAGENTA='\033[35m' +GREEN='\033[32m' +RESET='\033[0m' # Run cleanup commands in case script is interrupted -function cleanup() + +cleanup() { - BOLD='\033[1m' - RESET='\033[0m' echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" echo "⚠️ Script was abruptly exited. Running cleanup tasks." echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" pct unlock $CTID_TEMPLATE - if [ -f "/var/lib/vz/snippets/container-public-keys/$PUB_FILE" ]; then - rm -rf /var/lib/vz/snippets/container-public-keys/$PUB_FILE - fi - if [ -f "/var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE" ]; then - rm -rf /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE - fi - if [ -f "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" ]; then - rm -rf "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" - fi - if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then - rm -rf "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" - fi - + for file in \ + "/var/lib/vz/snippets/container-public-keys/$PUB_FILE" \ + "/var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE" \ + "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" \ + "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" + do + [ -f "$file" ] && rm -rf "$file" + done exit 1 } # Echo Container Details -function echoContainerDetails() { - BOLD='\033[1m' - BLUE='\033[34m' - MAGENTA='\033[35m' - GREEN='\033[32m' - RESET='\033[0m' - +echoContainerDetails() { echo -e "📦 ${BLUE}Container ID :${RESET} $CONTAINER_ID" echo -e "🌐 ${MAGENTA}Internal IP :${RESET} $CONTAINER_IP" echo -e "🔗 ${GREEN}Domain Name :${RESET} https://$CONTAINER_NAME.opensource.mieweb.org" echo -e "🛠️ ${BLUE}SSH Access :${RESET} ssh -p $SSH_PORT root@$CONTAINER_NAME.opensource.mieweb.org" - echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}${MAGENTA}NOTE: Additional background scripts are being ran in detached terminal sessions.${RESET}" + echo -e "${BOLD}${MAGENTA}Wait up to two minutes for all processes to complete.${RESET}" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}${BLUE}Still not working? Contact Max K. at maxklema@gmail.com${RESET}" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + } trap cleanup SIGINT SIGTERM SIGHUP @@ -70,19 +70,31 @@ LINUX_DISTRO="${18}" MULTI_COMPONENTS="${19}" ROOT_START_COMMAND="${20}" -if [ ${LINUX_DISTRO^^} == "DEBIAN" ]; then - PACKAGE_MANAGER="apt-get" - CTID_TEMPLATE="114" -elif [ "${LINUX_DISTRO^^}" == "ROCKY" ]; then - PACKAGE_MANAGER="dnf" - CTID_TEMPLATE="113" -fi +# Pick the correct template to clone ===== REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") +REPO_BASE_NAME_WITH_OWNER=$(echo "$PROJECT_REPOSITORY" | cut -d'/' -f4) -if [ "${GH_ACTION^^}" != "Y" ]; then - # Create the Container Clone +TEMPLATE_NAME="template-$REPO_BASE_NAME-$REPO_BASE_NAME_WITH_OWNER" +CTID_TEMPLATE=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$TEMPLATE_NAME" '$3 == name {print $1}') +case "${LINUX_DISTRO^^}" in + DEBIAN) PACKAGE_MANAGER="apt-get" ;; + ROCKY) PACKAGE_MANAGER="dnf" ;; +esac + +# If no template ID was provided, assign a default based on distro + +if [ -z "$CTID_TEMPLATE" ]; then + case "${LINUX_DISTRO^^}" in + DEBIAN) CTID_TEMPLATE="160" ;; + ROCKY) CTID_TEMPLATE="138" ;; + esac +fi + +# Create the Container Clone ==== + +if [ "${GH_ACTION^^}" != "Y" ]; then CONTAINER_ID=$(pvesh get /cluster/nextid) #Get the next available LXC ID echo "⏳ Cloning Container..." @@ -95,6 +107,7 @@ if [ "${GH_ACTION^^}" != "Y" ]; then echo "⏳ Setting Container Properties..." pct set $CONTAINER_ID \ --tags "$PROXMOX_USERNAME" \ + --tags "$LINUX_DISTRO" \ --onboot 1 > /dev/null 2>&1 pct start $CONTAINER_ID > /dev/null 2>&1 @@ -104,25 +117,21 @@ if [ "${GH_ACTION^^}" != "Y" ]; then # Get the Container IP Address and install some packages echo "⏳ Waiting for DHCP to allocate IP address to container..." - sleep 10 - - echo "⏳ Updating Packages.." - - pct exec $CONTAINER_ID -- bash -c "$PACKAGE_MANAGER upgrade -y" > /dev/null - pct exec $CONTAINER_ID -- bash -c "$PACKAGE_MANAGER install -y sudo git curl vim" > /dev/null - if [ -f "/var/lib/vz/snippets/container-public-keys/$PUB_FILE" ]; then - pct exec $CONTAINER_ID -- touch ~/.ssh/authorized_keys > /dev/null 2>&1 - pct exec $CONTAINER_ID -- bash -c "cat > ~/.ssh/authorized_keys"< /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 - rm -rf /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 - fi + sleep 5 # Set password inside the container - pct exec $CONTAINER_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 else CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$CONTAINER_NAME" '$3 == name {print $1}') fi +if [ -f "/var/lib/vz/snippets/container-public-keys/$PUB_FILE" ]; then + echo "⏳ Appending Public Key..." + pct exec $CONTAINER_ID -- touch ~/.ssh/authorized_keys > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "cat > ~/.ssh/authorized_keys"< /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 + rm -rf /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 +fi + CONTAINER_IP=$(pct exec $CONTAINER_ID -- hostname -I | awk '{print $1}') # Attempt to Automatically Deploy Project Inside Container @@ -131,12 +140,12 @@ if [ "${DEPLOY_ON_START^^}" == "Y" ]; then source /var/lib/vz/snippets/helper-scripts/deployOnStart.sh #cleanup - if [ -f "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" ]; then - rm -rf "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" > /dev/null 2>&1 - fi - if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then - rm -rf "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" > /dev/null 2>&1 - fi + for file in \ + "/var/lib/vz/snippets/container-env-vars/$ENV_BASE_FOLDER" \ + "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" + do + [ -f "$file" ] && rm -rf "$file" > /dev/null 2>&1 + done fi # Run Contianer Provision Script to add container to port_map.json @@ -150,84 +159,31 @@ fi SSH_PORT=$(iptables -t nat -S PREROUTING | grep "to-destination $CONTAINER_IP:22" | awk -F'--dport ' '{print $2}' | awk '{print $1}' | head -n 1 || true) -# Migrate to pve2 if Container ID is even and restart project ==== - -startProject() { - - RUNTIME="$1" - BUILD_CMD="$2" - START_CMD="$3" - COMP_DIR="$4" - - if [ "${RUNTIME^^}" == "NODEJS" ]; then - if [ "$BUILD_CMD" == "" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 - else - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && $BUILD_CMD && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 - fi - elif [ "${RUNTIME^^}" == "PYTHON" ]; then - if [ "$BUILD_CMD" == "" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\"" > /dev/null 2>&1 - else - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 - fi - fi - -} - -if (( $CONTAINER_ID % 2 == 0 )); then - if [ "${GH_ACTION^^}" != "Y" ]; then - pct stop $CONTAINER_ID > /dev/null 2>&1 - pct migrate $CONTAINER_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 - ssh root@10.15.0.5 "pct start $CONTAINER_ID" > /dev/null 2>&1 - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'chmod 700 ~/.bashrc'" > /dev/null 2>&1 # enable full R/W/X permissions - ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4" > /dev/null 2>&1 - if [ "${DEPLOY_ON_START^^}" == "Y" ]; then - if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then - for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do - START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') - BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - if [ "$BUILD" == "null" ]; then - BUILD="" - fi - startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" - done - if [ ! -z "$ROOT_START_COMMAND" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 - fi - else - startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." - fi - fi - ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2" - else - echoContainerDetails - echo "NOTE: Your Container needs to Migrate. Wait ~2 minutes before trying to SSH or navigate to the URL" - - CMD=( - bash /var/lib/vz/snippets/finish-migration.sh - "$CONTAINER_ID" - "$CONTAINER_NAME" - "$REPO_BASE_NAME" - "$REPO_BASE_NAME_WITH_OWNER" - "$SSH_PORT" - "$CONTAINER_IP" - "$PROJECT_ROOT" - "$ROOT_START_COMMAND" - "$DEPLOY_ON_START" - "$MULTI_COMPONENTS" - "$START_COMMAND" - "$BUILD_COMMAND" - "$RUNTIME_LANGUAGE" - ) - - # Safely quote each argument for the shell - QUOTED_CMD=$(printf ' %q' "${CMD[@]}") - - tmux new-session -d -s finish_migration "$QUOTED_CMD" - fi -fi - +# Output container details and start services if necessary ===== echoContainerDetails + +CMD=( +bash /var/lib/vz/snippets/start_services.sh +"$CONTAINER_ID" +"$CONTAINER_NAME" +"$REPO_BASE_NAME" +"$REPO_BASE_NAME_WITH_OWNER" +"$SSH_PORT" +"$CONTAINER_IP" +"$PROJECT_ROOT" +"$ROOT_START_COMMAND" +"$DEPLOY_ON_START" +"$MULTI_COMPONENTS" +"$START_COMMAND" +"$BUILD_COMMAND" +"$RUNTIME_LANGUAGE" +"$GH_ACTION" +"$PROJECT_BRANCH" +) + +# Safely quote each argument for the shell +QUOTED_CMD=$(printf ' %q' "${CMD[@]}") + +tmux new-session -d -s "$CONTAINER_NAME" "$QUOTED_CMD" +exit 0 \ No newline at end of file diff --git a/container creation/deployOnStart.sh b/container creation/deployOnStart.sh new file mode 100755 index 00000000..9873b2c0 --- /dev/null +++ b/container creation/deployOnStart.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Automation Script for attempting to automatically deploy projects and services on a container +# Last Modifided by Maxwell Klema on July 16th, 2025 +# ----------------------------------------------------- + +echo "🚀 Attempting Automatic Deployment" +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") + +# Clone github repository from correct branch ==== + +pct enter $CONTAINER_ID < /dev/null +EOF + +pct exec $CONTAINER_ID -- bash -c "chmod 700 ~/.bashrc" # enable full R/W/X permissions + +# Copy over ENV variables ==== + +ENV_BASE_FOLDER="/var/lib/vz/snippets/container-env-vars/${ENV_BASE_FOLDER}" + +if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then + for FILE in $ENV_BASE_FOLDER/*; do + FILE_BASENAME=$(basename "$FILE") + FILE_NAME="${FILE_BASENAME%.*}" + ENV_ROUTE=$(echo "$FILE_NAME" | tr '_' '/') # acts as the route to the correct folder to place .env file in. + + ENV_VARS=$(cat $ENV_BASE_FOLDER/$FILE_BASENAME) + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$ENV_ROUTE && echo "$ENV_VARS" > .env" > /dev/null 2>&1 + done +else + ENV_FOLDER_BASE_NAME=$(basename "$ENV_BASE_FOLDER") + ENV_VARS=$(cat $ENV_BASE_FOLDER/$ENV_FOLDER_BASE_NAME.txt) + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && echo "$ENV_VARS" > .env" > /dev/null 2>&1 +fi + +# Run Installation Commands ==== + +runInstallCommands() { + + RUNTIME="$1" + COMP_DIR="$2" + + if [ "${RUNTIME^^}" == "NODEJS" ]; then + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && sudo $INSTALL_CMD" > /dev/null 2>&1 + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + pct enter $CONTAINER_ID < /dev/null +cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && \ +python3 -m venv venv && source venv/bin/activate && \ +pip install --upgrade pip && \ +$INSTALL_CMD +EOF + fi +} + +if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then + for COMPONENT in $(echo "$RUNTIME_LANGUAGE" | jq -r 'keys[]'); do + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') #get runtime env + INSTALL_CMD=$(echo "$INSTALL_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') #get install command + if [ "$INSTALL_CMD" != "null" ]; then + runInstallCommands "$RUNTIME" "$COMPONENT" + fi + done +else + INSTALL_CMD=$INSTALL_COMMAND + runInstallCommands "$RUNTIME_LANGUAGE" "." +fi + +# Install Services ==== + +if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then + while read line; do + pct exec $CONTAINER_ID -- bash -c "$line" > /dev/null 2>&1 + done < "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" +fi \ No newline at end of file diff --git a/container maintenance/finish-migration.sh b/container maintenance/finish-migration.sh deleted file mode 100644 index 0969da8e..00000000 --- a/container maintenance/finish-migration.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# Script ran by a virtual terminal session to migrate a container -# Script is only ran on GH action workflows when runner disconnects due to migrations -# Last Modified by Maxwell Klema on July 22nd, 2025 -# ------------------------------------------------ - -CONTAINER_ID="$1" -CONTAINER_NAME="$2" -REPO_BASE_NAME="$3" -REPO_BASE_NAME_WITH_OWNER="$4" -SSH_PORT="$5" -CONTAINER_IP="$6" -PROJECT_ROOT="$7" -ROOT_START_COMMAND="$8" -DEPLOY_ON_START="$9" -MULTI_COMPONENT="${10}" -START_COMMAND="${11}" -BUILD_COMMAND="${12}" -RUNTIME_LANGUAGE="${13}" - -echo "$1 $2 $3 $4 $5 $6 $7 $8" -echo "Deploy on start: $9" >> "log.txt" -echo "Multi Component ${10}" >> "log.txt" -echo "Start command: ${11}" >> "log.txt" -echo "Build command: ${12}" >> "log.txt" -echo "Runtime language: ${13}" >> "log.txt" - -CONTAINER_NAME="${CONTAINER_NAME,,}" - -sleep 10 - -pct stop $CONTAINER_ID > /dev/null 2>&1 -pct migrate $CONTAINER_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 - -sleep 40 # wait for migration to finish (fix this later) - -ssh root@10.15.0.5 "pct start $CONTAINER_ID" -ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'chmod 700 ~/.bashrc'" # enable full R/W/X permissions -ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4" -ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- systemctl start github-runner" -ssh root@10.15.0.5 "pct start $CONTAINER_ID" > /dev/null 2>&1 -ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'chmod 700 ~/.bashrc'" > /dev/null 2>&1 # enable full R/W/X permissions -ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4" > /dev/null 2>&1 - -startProject() { - - RUNTIME="$1" - BUILD_CMD="$2" - START_CMD="$3" - COMP_DIR="$4" - - if [ "${RUNTIME^^}" == "NODEJS" ]; then - if [ "$BUILD_CMD" == "" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 - else - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && $BUILD_CMD && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 - fi - elif [ "${RUNTIME^^}" == "PYTHON" ]; then - if [ "$BUILD_CMD" == "" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\"" > /dev/null 2>&1 - else - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 - fi - fi - -} - -if [ "${DEPLOY_ON_START^^}" == "Y" ]; then - if [ "${MULTI_COMPONENT^^}" == "Y" ]; then - for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do - START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') - BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - if [ "$BUILD" == "null" ]; then - BUILD="" - fi - startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" - done - if [ ! -z "$ROOT_START_COMMAND" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 - fi - else - startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." - fi -fi - -ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2" \ No newline at end of file diff --git a/container maintenance/helper-scripts/create-template.sh b/container maintenance/helper-scripts/create-template.sh new file mode 100755 index 00000000..93cddb32 --- /dev/null +++ b/container maintenance/helper-scripts/create-template.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Creates a template of a LXC container +# Last modified by Maxwell Klema on July 23rd, 2025. +# -------------------------------------------------- + +echo "📝 Creating Container Template..." + +set -x + +if [ "${DEPLOY_ON_START^^}" != "Y" ] || [ "${GH_ACTION^^}" != "Y" ]; then + return 0 +fi + +DEFAULT_BRANCH=$(curl -s https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME | jq -r '.default_branch') + +if [ "$DEFAULT_BRANCH" != "$PROJECT_BRANCH" ]; then + return 0 +fi + +# Check if template already exists, and if it does, destroy it ===== + +TEMPLATE_NAME="template-$REPO_BASE_NAME-$REPO_BASE_NAME_WITH_OWNER" +TEMPLATE_CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$TEMPLATE_NAME" '$3 == name {print $1}') + +if [ ! -z "$TEMPLATE_CONTAINER_ID" ]; then + pct destroy $TEMPLATE_CONTAINER_ID +fi + +# Clone LXC container and convert it into a template ===== + +NEXT_ID=$(pvesh get /cluster/nextid) +pct clone $CONTAINER_ID $NEXT_ID \ + --hostname "$TEMPLATE_NAME" \ + --full true + +pct set $NEXT_ID --tags "$PROXMOX_USERNAME" +pct template $NEXT_ID \ No newline at end of file diff --git a/container maintenance/start_services.sh b/container maintenance/start_services.sh new file mode 100644 index 00000000..8b3c3a49 --- /dev/null +++ b/container maintenance/start_services.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# A script for cloning a Distro template, installing, and starting a runner on it. +# Last Modified by Maxwell Klema on July 20th, 2025 +# ------------------------------------------------ + +BOLD='\033[1m' +RESET='\033[0m' + +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +echo "🧬 Cloning a Template and installing a Runner" +echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + +# Validating Container Name ===== + +set +e + +source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh #Authenticate User +source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh #Ensure container does not exist. + +CONTAINER_EXISTS=$? + +if [ "$CONTAINER_EXISTS" != 1 ]; then + exit $CONTAINER_EXISTS; # Container is not free to user, either someone else owns it or the user owns it. +fi + +# Cloning Container Template and Setting it up ===== + +# Get correct template to clone and package manager +if [ ${LINUX_DISTRIBUTION^^} == "DEBIAN" ]; then + PACKAGE_MANAGER="apt" + CTID_TEMPLATE="160" +elif [ "${LINUX_DISTRIBUTION^^}" == "ROCKY" ]; then + PACKAGE_MANAGER="dnf" + CTID_TEMPLATE="138" +fi + +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") +REPO_BASE_NAME_WITH_OWNER=$(echo "$PROJECT_REPOSITORY" | cut -d'/' -f4) + +NEXT_ID=$(pvesh get /cluster/nextid) #Get the next available LXC ID + +# Create the Container Clone +echo "⏳ Cloning Container..." +pct clone $CTID_TEMPLATE $NEXT_ID \ + --hostname $CONTAINER_NAME \ + --full true > /dev/null 2>&1 + +# Set Container Options +echo "⏳ Setting Container Properties..." +pct set $NEXT_ID \ + --tags "$PROXMOX_USERNAME" \ + --onboot 1 > /dev/null 2>&1 + +pct start $NEXT_ID > /dev/null 2>&1 +pveum aclmod /vms/$NEXT_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser > /dev/null 2>&1 + +sleep 5 +echo "⏳ DHCP Allocating IP Address..." +CONTAINER_IP=$(pct exec $NEXT_ID -- hostname -I | awk '{print $1}') + +# Set password inside the container +pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 + +# Setting Up Github Runner ===== + +# Get Temporary Token +echo "🪙 Getting Authentication Token..." +AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/registration-token --header "Authorization: token $GITHUB_PAT") +TOKEN=$(echo "$AUTH_TOKEN_RESPONSE" | jq -r '.token') + +pct enter $NEXT_ID < /dev/null +export RUNNER_ALLOW_RUNASROOT=1 && \ +cd /actions-runner && ./config.sh --url $PROJECT_REPOSITORY --token $TOKEN --labels $CONTAINER_NAME +EOF + +# Generate RSA Keys ===== + +echo "🔑 Generating RSA Key Pair..." +pct exec $NEXT_ID -- bash -c "ssh-keygen -t rsa -N '' -f /root/.ssh/id_rsa -q" +PUB_KEY=$(pct exec $NEXT_ID -- bash -c "cat /root/.ssh/id_rsa.pub") + +# Place public key in all necessary authorized_keys files +echo "$PUB_KEY" >> /home/create-container/.ssh/authorized_keys +echo "$PUB_KEY" >> /home/update-container/.ssh/authorized_keys +echo "$PUB_KEY" >> /home/delete-container/.ssh/authorized_keys +echo "$PUB_KEY" >> /home/container-exists/.ssh/authorized_keys + +ssh root@10.15.234.122 "echo \"$PUB_KEY\" >> /root/.ssh/authorized_keys" + +echo "🔑 Creating Service File..." +pct exec $NEXT_ID -- bash -c "cat < /etc/systemd/system/github-runner.service +[Unit] +Description=GitHub Actions Runner +After=network.target + +[Service] +Type=simple +WorkingDirectory=/actions-runner +Environment=\"RUNNER_ALLOW_RUNASROOT=1\" +ExecStart=/actions-runner/run.sh +Restart=always + +[Install] +WantedBy=multi-user.target +EOF" + +pct exec $NEXT_ID -- systemctl daemon-reload +pct exec $NEXT_ID -- systemctl enable github-runner +pct exec $NEXT_ID -- systemctl start github-runner + +exit 3 From f71efeb01ba2b3304a0ba6861d06688f0f2f14c8 Mon Sep 17 00:00:00 2001 From: maxklema Date: Mon, 28 Jul 2025 12:22:40 -0400 Subject: [PATCH 21/25] CI/CD Github Action Scripts + Helper Script + Container Creation Script Updates + Bug Fixes --- container creation/create-container.sh | 26 +- container creation/deployOnStart.sh | 9 +- container creation/setup-runner.sh | 48 ++-- .../check-container-exists.sh | 7 +- .../helper-scripts/create-template.sh | 35 ++- .../helper-scripts/delete-runner.sh | 4 +- container maintenance/start_services.sh | 252 ++++++++++-------- container maintenance/update-container.sh | 96 ++++++- 8 files changed, 310 insertions(+), 167 deletions(-) diff --git a/container creation/create-container.sh b/container creation/create-container.sh index a4d47423..dd485893 100644 --- a/container creation/create-container.sh +++ b/container creation/create-container.sh @@ -60,15 +60,16 @@ DEPLOY_ON_START="$8" PROJECT_REPOSITORY="$9" PROJECT_BRANCH="${10}" PROJECT_ROOT="${11}" -INSTALL_COMMAND="${12}" -BUILD_COMMAND="${13}" -START_COMMAND="${14}" -RUNTIME_LANGUAGE="${15}" +INSTALL_COMMAND=$(echo "${12}" | base64 -d) +BUILD_COMMAND=$(echo "${13}" | base64 -d) +START_COMMAND=$(echo "${14}" | base64 -d) +RUNTIME_LANGUAGE=$(echo "${15}" | base64 -d) ENV_BASE_FOLDER="${16}" SERVICES_BASE_FILE="${17}" LINUX_DISTRO="${18}" MULTI_COMPONENTS="${19}" ROOT_START_COMMAND="${20}" +GITHUB_PAT="${21}" # Pick the correct template to clone ===== @@ -148,6 +149,10 @@ if [ "${DEPLOY_ON_START^^}" == "Y" ]; then done fi +# Create Log File ==== + +pct exec $CONTAINER_ID -- bash -c "cd /root && touch container-updates.log" + # Run Contianer Provision Script to add container to port_map.json if [ -f "/var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE" ]; then @@ -162,7 +167,11 @@ SSH_PORT=$(iptables -t nat -S PREROUTING | grep "to-destination $CONTAINER_IP:22 # Output container details and start services if necessary ===== echoContainerDetails - + +BUILD_COMMAND_B64=$(echo -n "$BUILD_COMMAND" | base64) +RUNTIME_LANGUAGE_B64=$(echo -n "$RUNTIME_LANGUAGE" | base64) +START_COMMAND_B64=$(echo -n "$START_COMMAND" | base64) + CMD=( bash /var/lib/vz/snippets/start_services.sh "$CONTAINER_ID" @@ -175,11 +184,12 @@ bash /var/lib/vz/snippets/start_services.sh "$ROOT_START_COMMAND" "$DEPLOY_ON_START" "$MULTI_COMPONENTS" -"$START_COMMAND" -"$BUILD_COMMAND" -"$RUNTIME_LANGUAGE" +"$START_COMMAND_B64" +"$BUILD_COMMAND_B64" +"$RUNTIME_LANGUAGE_B64" "$GH_ACTION" "$PROJECT_BRANCH" +"$GITHUB_PAT" ) # Safely quote each argument for the shell diff --git a/container creation/deployOnStart.sh b/container creation/deployOnStart.sh index 9873b2c0..ae1bb486 100755 --- a/container creation/deployOnStart.sh +++ b/container creation/deployOnStart.sh @@ -8,9 +8,16 @@ REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") # Clone github repository from correct branch ==== +echo "Repo base name: $REPO_BASE_NAME" + pct enter $CONTAINER_ID < /dev/null +git clone $PROJECT_REPOSITORY && git checkout $PROJECT_BRANCH > /dev/null +else +cd /root/$REPO_BASE_NAME && git fetch && git pull && \ +git checkout $PROJECT_BRANCH +fi EOF pct exec $CONTAINER_ID -- bash -c "chmod 700 ~/.bashrc" # enable full R/W/X permissions diff --git a/container creation/setup-runner.sh b/container creation/setup-runner.sh index 33261665..53bcf442 100644 --- a/container creation/setup-runner.sh +++ b/container creation/setup-runner.sh @@ -12,8 +12,6 @@ echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━ # Validating Container Name ===== -set +e - source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh #Authenticate User source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh #Ensure container does not exist. @@ -25,14 +23,25 @@ fi # Cloning Container Template and Setting it up ===== -# Get correct template to clone and package manager -if [ ${LINUX_DISTRIBUTION^^} == "DEBIAN" ]; then - PACKAGE_MANAGER="apt" - CTID_TEMPLATE="114" -elif [ "${LINUX_DISTRIBUTION^^}" == "ROCKY" ]; then - PACKAGE_MANAGER="dnf" - CTID_TEMPLATE="113" -fi +REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") +REPO_BASE_NAME_WITH_OWNER=$(echo "$PROJECT_REPOSITORY" | cut -d'/' -f4) + +TEMPLATE_NAME="template-$REPO_BASE_NAME-$REPO_BASE_NAME_WITH_OWNER" +CTID_TEMPLATE=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$TEMPLATE_NAME" '$3 == name {print $1}') + +case "${LINUX_DISTRIBUTION^^}" in + DEBIAN) PACKAGE_MANAGER="apt-get" ;; + ROCKY) PACKAGE_MANAGER="dnf" ;; +esac + +# If no template ID was provided, assign a default based on distro + +if [ -z "$CTID_TEMPLATE" ]; then + case "${LINUX_DISTRIBUTION^^}" in + DEBIAN) CTID_TEMPLATE="160" ;; + ROCKY) CTID_TEMPLATE="138" ;; + esac +fi REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") REPO_BASE_NAME_WITH_OWNER=$(echo "$PROJECT_REPOSITORY" | cut -d'/' -f4) @@ -58,15 +67,8 @@ sleep 5 echo "⏳ DHCP Allocating IP Address..." CONTAINER_IP=$(pct exec $NEXT_ID -- hostname -I | awk '{print $1}') -# Set password inside the container and install some pacakages -echo "📦 Updating Packages..." +# Set password inside the container pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 -pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER upgrade -y" > /dev/null > /dev/null 2>&1 -if [ "${LINUX_DISTRIBUTION^^}" == "DEBIAN" ]; then - pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER install -y sudo git curl vim tar tmux sshpass jq" > /dev/null 2>&1 -elif [ "${LINUX_DISTRIBUTION^^}" == "ROCKY" ]; then - pct exec $NEXT_ID -- bash -c "$PACKAGE_MANAGER install -y sudo git curl vim tar tmux libicu perl-Digest-SHA sshpass jq" > /dev/null 2>&1 -fi # Setting Up Github Runner ===== @@ -76,12 +78,12 @@ AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repo TOKEN=$(echo "$AUTH_TOKEN_RESPONSE" | jq -r '.token') pct enter $NEXT_ID < /dev/null -mkdir actions-runner && cd actions-runner && \ -curl -o actions-runner-linux-x64-2.326.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.326.0/actions-runner-linux-x64-2.326.0.tar.gz && \ -echo "9c74af9b4352bbc99aecc7353b47bcdfcd1b2a0f6d15af54a99f54a0c14a1de8 actions-runner-linux-x64-2.326.0.tar.gz" | shasum -a 256 -c && \ -tar xzf ./actions-runner-linux-x64-2.326.0.tar.gz && \ +rm -rf /root/container-updates.log | true && \ +cd /actions-runner && export RUNNER_ALLOW_RUNASROOT=1 && \ +export runProcess=\$(ps aux | grep run.sh | awk '{print \$2}' | head -n 1) && kill -9 \$runProcess || true && \ +rm -rf .runner .credentials && rm -rf _work/* /var/log/runner/* && \ export RUNNER_ALLOW_RUNASROOT=1 && \ -./config.sh --url $PROJECT_REPOSITORY --token $TOKEN --labels $CONTAINER_NAME +./config.sh --url $PROJECT_REPOSITORY --token $TOKEN --labels $CONTAINER_NAME --name $CONTAINER_NAME EOF # Generate RSA Keys ===== diff --git a/container maintenance/check-container-exists.sh b/container maintenance/check-container-exists.sh index d8ca9710..c2aded15 100644 --- a/container maintenance/check-container-exists.sh +++ b/container maintenance/check-container-exists.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to check if a container exists -# Last Modified by Maxwell Klema on July 22nd, 2025 +# Last Modified by Maxwell Klema on July 13th, 2025 # ----------------------------------------------------- RESET="\033[0m" @@ -25,15 +25,14 @@ fi REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") # Check if repository folder is present. - if [ "$PVE1" == "true" ]; then - if pct exec $CONTAINER_ID -- test -d /root/$REPO_BASE_NAME; then + if pct exec $CONTAINER_ID -- test -f /root/container-updates.log; then exit 2; # Update Repository else exit 0; # Clone Repository fi else - if ssh 10.15.0.5 "pct exec $CONTAINER_ID -- test -d /root/$REPO_BASE_NAME"; then + if ssh 10.15.0.5 "pct exec $CONTAINER_ID -- test -f /root/container-updates.log"; then exit 2; # Update Repository else exit 0; # Clone Repository diff --git a/container maintenance/helper-scripts/create-template.sh b/container maintenance/helper-scripts/create-template.sh index 93cddb32..2f80b706 100755 --- a/container maintenance/helper-scripts/create-template.sh +++ b/container maintenance/helper-scripts/create-template.sh @@ -3,10 +3,6 @@ # Last modified by Maxwell Klema on July 23rd, 2025. # -------------------------------------------------- -echo "📝 Creating Container Template..." - -set -x - if [ "${DEPLOY_ON_START^^}" != "Y" ] || [ "${GH_ACTION^^}" != "Y" ]; then return 0 fi @@ -17,21 +13,44 @@ if [ "$DEFAULT_BRANCH" != "$PROJECT_BRANCH" ]; then return 0 fi +echo "📝 Creating Container Template..." + # Check if template already exists, and if it does, destroy it ===== TEMPLATE_NAME="template-$REPO_BASE_NAME-$REPO_BASE_NAME_WITH_OWNER" TEMPLATE_CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$TEMPLATE_NAME" '$3 == name {print $1}') if [ ! -z "$TEMPLATE_CONTAINER_ID" ]; then - pct destroy $TEMPLATE_CONTAINER_ID + pct destroy $TEMPLATE_CONTAINER_ID | true fi # Clone LXC container and convert it into a template ===== NEXT_ID=$(pvesh get /cluster/nextid) -pct clone $CONTAINER_ID $NEXT_ID \ - --hostname "$TEMPLATE_NAME" \ - --full true +if (( $CONTAINER_ID % 2 == 101 )); then + ssh root@10.15.0.5 " + pct clone $CONTAINER_ID $NEXT_ID \ + --hostname "$TEMPLATE_NAME" \ + --full true + pct migrate $NEXT_ID intern-phxdc-pve1 --target-storage containers-pve1 + " > /dev/null 2>&1 +else + pct clone $CONTAINER_ID $NEXT_ID \ + --hostname "$TEMPLATE_NAME" \ + --full true +fi + +AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/registration-token --header "Authorization: token $GITHUB_PAT") +TOKEN=$(echo "$AUTH_TOKEN_RESPONSE" | jq -r '.token') + +# Remove rsa keys ==== +pct start $NEXT_ID +pct enter $NEXT_ID < /dev/null 2>&1 + ssh root@10.15.0.5 "pct stop $CONTAINER_ID || true && pct destroy $CONTAINER_ID" > /dev/null 2>&1 else ssh root@10.15.0.5 "pct destroy $CONTAINER_ID" > /dev/null 2>&1 fi diff --git a/container maintenance/start_services.sh b/container maintenance/start_services.sh index 8b3c3a49..165a96a0 100644 --- a/container maintenance/start_services.sh +++ b/container maintenance/start_services.sh @@ -1,111 +1,149 @@ #!/bin/bash -# A script for cloning a Distro template, installing, and starting a runner on it. -# Last Modified by Maxwell Klema on July 20th, 2025 +# Script ran by a virtual terminal session to start services and migrate a container +# Script is only ran on GH action workflows when runner disconnects +# Last Modified by Maxwell Klema on July 23rd, 2025 # ------------------------------------------------ -BOLD='\033[1m' -RESET='\033[0m' - -echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -echo "🧬 Cloning a Template and installing a Runner" -echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" - -# Validating Container Name ===== - -set +e - -source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh #Authenticate User -source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh #Ensure container does not exist. - -CONTAINER_EXISTS=$? - -if [ "$CONTAINER_EXISTS" != 1 ]; then - exit $CONTAINER_EXISTS; # Container is not free to user, either someone else owns it or the user owns it. -fi - -# Cloning Container Template and Setting it up ===== - -# Get correct template to clone and package manager -if [ ${LINUX_DISTRIBUTION^^} == "DEBIAN" ]; then - PACKAGE_MANAGER="apt" - CTID_TEMPLATE="160" -elif [ "${LINUX_DISTRIBUTION^^}" == "ROCKY" ]; then - PACKAGE_MANAGER="dnf" - CTID_TEMPLATE="138" -fi - -REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") -REPO_BASE_NAME_WITH_OWNER=$(echo "$PROJECT_REPOSITORY" | cut -d'/' -f4) - -NEXT_ID=$(pvesh get /cluster/nextid) #Get the next available LXC ID - -# Create the Container Clone -echo "⏳ Cloning Container..." -pct clone $CTID_TEMPLATE $NEXT_ID \ - --hostname $CONTAINER_NAME \ - --full true > /dev/null 2>&1 - -# Set Container Options -echo "⏳ Setting Container Properties..." -pct set $NEXT_ID \ - --tags "$PROXMOX_USERNAME" \ - --onboot 1 > /dev/null 2>&1 - -pct start $NEXT_ID > /dev/null 2>&1 -pveum aclmod /vms/$NEXT_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser > /dev/null 2>&1 - -sleep 5 -echo "⏳ DHCP Allocating IP Address..." -CONTAINER_IP=$(pct exec $NEXT_ID -- hostname -I | awk '{print $1}') - -# Set password inside the container -pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 - -# Setting Up Github Runner ===== - -# Get Temporary Token -echo "🪙 Getting Authentication Token..." -AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/registration-token --header "Authorization: token $GITHUB_PAT") -TOKEN=$(echo "$AUTH_TOKEN_RESPONSE" | jq -r '.token') - -pct enter $NEXT_ID < /dev/null -export RUNNER_ALLOW_RUNASROOT=1 && \ -cd /actions-runner && ./config.sh --url $PROJECT_REPOSITORY --token $TOKEN --labels $CONTAINER_NAME +set -x +CONTAINER_ID="$1" +CONTAINER_NAME="$2" +REPO_BASE_NAME="$3" +REPO_BASE_NAME_WITH_OWNER="$4" +SSH_PORT="$5" +CONTAINER_IP="$6" +PROJECT_ROOT="$7" +ROOT_START_COMMAND="$8" +DEPLOY_ON_START="$9" +MULTI_COMPONENT="${10}" +START_COMMAND=$(echo "${11}" | base64 -d) +BUILD_COMMAND=$(echo "${12}" | base64 -d) +RUNTIME_LANGUAGE=$(echo "${13}" | base64 -d) +GH_ACTION="${14}" +PROJECT_BRANCH="${15}" +GITHUB_PAT="${16}" +UPDATE_CONTAINER="${17}" +CONTAINER_NAME="${CONTAINER_NAME,,}" + +sleep 3 +pct stop $CONTAINER_ID > /dev/null 2>&1 + +echo "$START_COMMAND" +echo "$BUILD_COMMAND" +echo "$RUNTIME_LANGUAGE" + +sleep 10 + + +# Create template if on default branch ==== +# source /var/lib/vz/snippets/helper-scripts/create-template.sh + +if (( $CONTAINER_ID % 2 == 0 )); then + + if [ "$UPDATE_CONTAINER" != "true" ]; then + pct migrate $CONTAINER_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 + sleep 40 # wait for migration to finish (fix this later) + fi + + ssh root@10.15.0.5 "pct start $CONTAINER_ID" + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'chmod 700 ~/.bashrc'" # enable full R/W/X permissions + ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4" + + if [ "${GH_ACTION^^}" == "Y" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- systemctl start github-runner" + fi + + startProject() { + + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" + + if [ "${RUNTIME^^}" == "NODEJS" ]; then + if [ "$BUILD_CMD" == "" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 + else + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && $BUILD_CMD && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 + fi + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + if [ "$BUILD_CMD" == "" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\"" > /dev/null 2>&1 + else + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 + fi + fi + + } + + if [ "${DEPLOY_ON_START^^}" == "Y" ]; then + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" + done + if [ ! -z "$ROOT_START_COMMAND" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi + else + startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + fi + fi + ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2" + +# PVE 1 +else + pct start $CONTAINER_ID | true + if [ "${GH_ACTION^^}" == "Y" ]; then + pct exec $CONTAINER_ID -- bash -c "systemctl start github-runner" + fi + + startComponent() { + + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" + + if [ "${RUNTIME^^}" == "NODEJS" ]; then + if [ "$BUILD_CMD" == "" ]; then + pct exec $CONTAINER_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD'" > /dev/null 2>&1 + else + pct enter $CONTAINER_ID < /dev/null +export PATH=\$PATH:/usr/local/bin && \ +$BUILD_CMD || true && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD' EOF - -# Generate RSA Keys ===== - -echo "🔑 Generating RSA Key Pair..." -pct exec $NEXT_ID -- bash -c "ssh-keygen -t rsa -N '' -f /root/.ssh/id_rsa -q" -PUB_KEY=$(pct exec $NEXT_ID -- bash -c "cat /root/.ssh/id_rsa.pub") - -# Place public key in all necessary authorized_keys files -echo "$PUB_KEY" >> /home/create-container/.ssh/authorized_keys -echo "$PUB_KEY" >> /home/update-container/.ssh/authorized_keys -echo "$PUB_KEY" >> /home/delete-container/.ssh/authorized_keys -echo "$PUB_KEY" >> /home/container-exists/.ssh/authorized_keys - -ssh root@10.15.234.122 "echo \"$PUB_KEY\" >> /root/.ssh/authorized_keys" - -echo "🔑 Creating Service File..." -pct exec $NEXT_ID -- bash -c "cat < /etc/systemd/system/github-runner.service -[Unit] -Description=GitHub Actions Runner -After=network.target - -[Service] -Type=simple -WorkingDirectory=/actions-runner -Environment=\"RUNNER_ALLOW_RUNASROOT=1\" -ExecStart=/actions-runner/run.sh -Restart=always - -[Install] -WantedBy=multi-user.target -EOF" - -pct exec $NEXT_ID -- systemctl daemon-reload -pct exec $NEXT_ID -- systemctl enable github-runner -pct exec $NEXT_ID -- systemctl start github-runner - -exit 3 + fi + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + if [ "$BUILD_CMD" == "" ]; then + pct exec $CONTAINER_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'" > /dev/null 2>&1 + else + pct exec $CONTAINER_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'" > /dev/null 2>&1 + fi + fi + } + + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 > /dev/null #temporarily bump up container resources for computation hungry processes (e.g. meteor) + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + + startComponent "$RUNTIME" "$BUILD" "$START" "$COMPONENT" + done + if [ ! -z "$ROOT_START_COMMAND" ]; then + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" > /dev/null 2>&1 + fi + else + startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + fi + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 > /dev/null +fi \ No newline at end of file diff --git a/container maintenance/update-container.sh b/container maintenance/update-container.sh index aa587318..7fc9342a 100644 --- a/container maintenance/update-container.sh +++ b/container maintenance/update-container.sh @@ -65,7 +65,7 @@ done # # Get Project Root Directroy -if [ "$PROJECT_ROOT" == "" ]; then +if [ "$PROJECT_ROOT" == "." ] || [ "$PROJECT_ROOT" == "" ]; then PROJECT_ROOT="/" fi @@ -78,12 +78,45 @@ while [ "$VALID_PROJECT_ROOT" == "false" ]; do done REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") +REPO_BASE_NAME_WITH_OWNER=$(echo "$PROJECT_REPOSITORY" | cut -d'/' -f4) if [ "$PROJECT_ROOT" == "" ] || [ "$PROJECT_ROOT" == "/" ]; then PROJECT_ROOT="." fi -# Update Container with New Contents from repository +# Install Services ==== + +echo "🛎️ Installing Services..." + +# SERVICE_COMMANDS=$(ssh -o SendEnv="LINUX_DISTRIBUTION SERVICES CUSTOM_SERVICES REQUIRE_SERVICES" \ +# root@10.15.234.122 \ +# "/root/bin/deployment-scripts/gatherServices.sh true") + +# echo "$SERVICE_COMMANDS" | while read -r line; do +# pct exec $CONTAINER_ID -- bash -c "$line | true" > /dev/null 2>&1 +# done + +# Clone repository if needed ==== + +if (( "$CONTAINER_ID" % 2 == 0 )); then + ssh root@10.15.0.5 " + pct enter $CONTAINER_ID < /dev/null +fi +EOF + " +else + pct enter $CONTAINER_ID < /dev/null +fi +EOF +fi + +# Update Container with New Contents from repository ===== startComponentPVE1() { @@ -96,14 +129,12 @@ startComponentPVE1() { if [ "${RUNTIME^^}" == "NODEJS" ]; then pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null 2>&1 - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD" > /dev/null 2>&1 - pct exec $CONTAINER_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD'" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD && $BUILD_CMD" > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 elif [ "${RUNTIME^^}" == "PYTHON" ]; then pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null 2>&1 - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $INSTALL_CMD' && '$BUILD_CMD" > /dev/null 2>&1 - pct exec $CONTAINER_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $INSTALL_CMD && $BUILD_CMD" > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 fi } @@ -121,7 +152,6 @@ startComponentPVE2() { pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 - pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"' > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 " elif [ "${RUNTIME^^}" == "PYTHON" ]; then @@ -129,7 +159,6 @@ startComponentPVE2() { pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 - pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\" > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 " fi @@ -153,7 +182,7 @@ if [ "${MULTI_COMPONENT^^}" == "Y" ]; then fi done if [ ! -z "$ROOT_START_COMMAND" ]; then - if (( "$CONTAINER_ID" % 2 == 0 )); then + if (( $CONTAINER_ID % 2 == 0 )); then ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" else pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" @@ -161,12 +190,53 @@ if [ "${MULTI_COMPONENT^^}" == "Y" ]; then fi # startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." else - if (( "$CONTAINER_ID" % 2 == 0 )); then - startComponentPVE2 "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + if (( $CONTAINER_ID % 2 == 0 )); then + startComponentPVE2 "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." "$INSTALL_COMMAND" else - startComponentPVE1 "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + startComponentPVE1 "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." "$INSTALL_COMMAND" fi fi +# Update Log File + +if (( "$CONTAINER_ID" % 2 == 0 )); then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'echo \"[$(date)]\" >> /root/container-updates.log'" +else + pct exec $CONTAINER_ID -- bash -c "echo \"[$(date)]\" >> /root/container-updates.log" +fi + +# Create new template if on default branch ===== + +UPDATE_CONTAINER="true" +BUILD_COMMAND_B64=$(echo -n "$BUILD_COMMAND" | base64) +RUNTIME_LANGUAGE_B64=$(echo -n "$RUNTIME_LANGUAGE" | base64) +START_COMMAND_B64=$(echo -n "$START_COMMAND" | base64) + +CMD=( +bash /var/lib/vz/snippets/start_services.sh +"$CONTAINER_ID" +"$CONTAINER_NAME" +"$REPO_BASE_NAME" +"$REPO_BASE_NAME_WITH_OWNER" +"$SSH_PORT" +"$CONTAINER_IP" +"$PROJECT_ROOT" +"$ROOT_START_COMMAND" +"$DEPLOY_ON_START" +"$MULTI_COMPONENT" +"$START_COMMAND_B64" +"$BUILD_COMMAND_B64" +"$RUNTIME_LANGUAGE_B64" +"$GH_ACTION" +"$PROJECT_BRANCH" +"$GITHUB_PAT" +"$UPDATE_CONTAINER" +) + +# Safely quote each argument for the shell +QUOTED_CMD=$(printf ' %q' "${CMD[@]}") + +tmux new-session -d -s "$CONTAINER_NAME" "$QUOTED_CMD" echo "✅ Container $CONTAINER_ID has been updated with new contents from branch \"$PROJECT_BRANCH\" on repository \"$PROJECT_REPOSITORY\"." -exit 0 \ No newline at end of file +exit 0 + From b9ce00da1c83bd88313a6740244f78369619141d Mon Sep 17 00:00:00 2001 From: Maxwell Klema <80615123+maxklema@users.noreply.github.com> Date: Tue, 5 Aug 2025 16:01:00 -0400 Subject: [PATCH 22/25] CI/CD Updates and File Re-Organization (#2) * LDAP configuration and prune scripts * proxmox deployment changes * updated container-creation scripts + re-organization * READMEs in each directory, re-organization, updated ci-cd files * READMEs in each directory, re-organization, updated ci-cd files * proxmox launchpad submodule in ci-cd automation * proxmox launchpad submodule * proxmox launchpad submodule --- .gitmodules | 3 + ci-cd automation/README.md | 1 + .../check-container-exists.sh | 12 +- .../delete-container.sh | 2 +- .../helper-scripts/PVE_user_authentication.sh | 18 +- .../helper-scripts/create-template.sh | 4 +- .../helper-scripts/delete-runner.sh | 2 + .../helper-scripts/repository_status.sh | 0 .../verify_container_ownership.sh | 6 +- ci-cd automation/proxmox-launchpad/.gitignore | 0 ci-cd automation/proxmox-launchpad/README.md | 308 +++++++++++ ci-cd automation/proxmox-launchpad/action.yml | 494 ++++++++++++++++++ .../update-container.sh | 88 ++-- container creation/README.md | 1 + container creation/configureLDAP.sh | 34 ++ container creation/create-container.sh | 80 +-- container creation/deployOnStart.sh | 32 +- .../deployment-scripts}/gatherEnvVars.sh | 30 +- .../deployment-scripts}/gatherRuntimeLangs.sh | 25 +- .../deployment-scripts/gatherServices.sh | 158 ++++++ .../gatherSetupCommands.sh | 8 +- container creation/get-deployment-details.sh | 223 ++++---- .../get-lxc-container-details.sh | 269 +++++----- .../protocols/master_protocol_list.txt | 145 +++++ .../register-container.sh | 0 ...rvice_map.json => service_map_debian.json} | 48 +- .../services/service_map_rocky.json | 99 ++++ container creation/setup-runner.sh | 57 +- .../start_services.sh | 68 +-- container registration/register_proxy_hook.sh | 17 - dnsmasq service/README.md | 1 + gateway/README.md | 1 + .../extract-fingerprint.sh | 0 gateway/prune_iptables.sh | 173 ++++++ gateway/prune_temp_files.sh | 67 +++ nginx reverse proxy/README.md | 1 + .../nginx.conf | 0 .../port_map.js | 0 .../reverse_proxy.conf | 0 proxmox-launchpad | 1 + 40 files changed, 1991 insertions(+), 485 deletions(-) create mode 100644 .gitmodules create mode 100644 ci-cd automation/README.md rename {container maintenance => ci-cd automation}/check-container-exists.sh (70%) rename {container maintenance => ci-cd automation}/delete-container.sh (91%) rename {container maintenance => ci-cd automation}/helper-scripts/PVE_user_authentication.sh (58%) mode change 100644 => 100755 rename {container maintenance => ci-cd automation}/helper-scripts/create-template.sh (85%) rename {container maintenance => ci-cd automation}/helper-scripts/delete-runner.sh (97%) mode change 100644 => 100755 rename {container maintenance => ci-cd automation}/helper-scripts/repository_status.sh (100%) mode change 100644 => 100755 rename {container maintenance => ci-cd automation}/helper-scripts/verify_container_ownership.sh (87%) mode change 100644 => 100755 create mode 100644 ci-cd automation/proxmox-launchpad/.gitignore create mode 100644 ci-cd automation/proxmox-launchpad/README.md create mode 100644 ci-cd automation/proxmox-launchpad/action.yml rename {container maintenance => ci-cd automation}/update-container.sh (77%) create mode 100644 container creation/README.md create mode 100755 container creation/configureLDAP.sh rename {deployment-scripts => container creation/deployment-scripts}/gatherEnvVars.sh (74%) rename {deployment-scripts => container creation/deployment-scripts}/gatherRuntimeLangs.sh (63%) mode change 100644 => 100755 create mode 100755 container creation/deployment-scripts/gatherServices.sh rename {deployment-scripts => container creation/deployment-scripts}/gatherSetupCommands.sh (83%) create mode 100644 container creation/protocols/master_protocol_list.txt rename {container registration => container creation}/register-container.sh (100%) rename container creation/services/{service_map.json => service_map_debian.json} (56%) create mode 100644 container creation/services/service_map_rocky.json rename {container maintenance => container creation}/start_services.sh (59%) delete mode 100644 container registration/register_proxy_hook.sh create mode 100644 dnsmasq service/README.md create mode 100644 gateway/README.md rename {jump server => gateway}/extract-fingerprint.sh (100%) create mode 100644 gateway/prune_iptables.sh create mode 100644 gateway/prune_temp_files.sh create mode 100644 nginx reverse proxy/README.md rename {nginx proxy => nginx reverse proxy}/nginx.conf (100%) rename {nginx proxy => nginx reverse proxy}/port_map.js (100%) rename {nginx proxy => nginx reverse proxy}/reverse_proxy.conf (100%) create mode 160000 proxmox-launchpad diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..c8f2499b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "proxmox-launchpad"] + path = proxmox-launchpad + url = https://github.com/maxklema/proxmox-launchpad.git diff --git a/ci-cd automation/README.md b/ci-cd automation/README.md new file mode 100644 index 00000000..b8b97510 --- /dev/null +++ b/ci-cd automation/README.md @@ -0,0 +1 @@ +# CI/CD Automation \ No newline at end of file diff --git a/container maintenance/check-container-exists.sh b/ci-cd automation/check-container-exists.sh similarity index 70% rename from container maintenance/check-container-exists.sh rename to ci-cd automation/check-container-exists.sh index c2aded15..77c8fd8f 100644 --- a/container maintenance/check-container-exists.sh +++ b/ci-cd automation/check-container-exists.sh @@ -1,8 +1,16 @@ #!/bin/bash -# Script to check if a container exists +# Script to check if a container exists, and if so, whether it needs to be updated or cloned. # Last Modified by Maxwell Klema on July 13th, 2025 # ----------------------------------------------------- +outputError() { + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}${MAGENTA}❌ Script Failed. Exiting... ${RESET}" + echo -e "$2" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + exit $1 +} + RESET="\033[0m" BOLD="\033[1m" MAGENTA='\033[35m' @@ -11,13 +19,11 @@ echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━ echo -e "${BOLD}${MAGENTA}🔎 Check Container Exists ${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -set +e TYPE_RUNNER="true" source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh STATUS=$? - if [ "$STATUS" != 0 ]; then exit 1; fi diff --git a/container maintenance/delete-container.sh b/ci-cd automation/delete-container.sh similarity index 91% rename from container maintenance/delete-container.sh rename to ci-cd automation/delete-container.sh index c3538ce3..d46ce23b 100644 --- a/container maintenance/delete-container.sh +++ b/ci-cd automation/delete-container.sh @@ -25,5 +25,5 @@ QUOTED_CMD=$(printf ' %q' "${CMD[@]}") tmux new-session -d -s delete-runner "$QUOTED_CMD" -echo "✅ Container with name \"$CONTAINER_NAME\" will been permanently deleted." +echo "✅ Container with name \"$CONTAINER_NAME\" will be permanently deleted." exit 0 # Container Deleted Successfully \ No newline at end of file diff --git a/container maintenance/helper-scripts/PVE_user_authentication.sh b/ci-cd automation/helper-scripts/PVE_user_authentication.sh old mode 100644 new mode 100755 similarity index 58% rename from container maintenance/helper-scripts/PVE_user_authentication.sh rename to ci-cd automation/helper-scripts/PVE_user_authentication.sh index c751aadc..a7b84f34 --- a/container maintenance/helper-scripts/PVE_user_authentication.sh +++ b/ci-cd automation/helper-scripts/PVE_user_authentication.sh @@ -15,21 +15,9 @@ if [ -z "$PROXMOX_PASSWORD" ]; then fi USER_AUTHENTICATED=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateUser \"$PROXMOX_USERNAME\" \"$PROXMOX_PASSWORD\"") -RETRIES=3 -while [ $USER_AUTHENTICATED == 'false' ]; do - if [ $RETRIES -gt 0 ]; then - echo "❌ Authentication Failed. Try Again" - read -p "Enter Proxmox Username → " PROXMOX_USERNAME - read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD - echo "" - - USER_AUTHENTICATED=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateUser \"$PROXMOX_USERNAME\" \"$PROXMOX_PASSWORD\"") - RETRIES=$(($RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 2 - fi -done +if [ $USER_AUTHENTICATED == 'false' ]; then + outputError 1 "Your Proxmox account, $PROXMOX_USERNAME@pve, was not authenticated. Retry with valid credentials." +fi echo "🎉 Your proxmox account, $PROXMOX_USERNAME@pve, has been authenticated" \ No newline at end of file diff --git a/container maintenance/helper-scripts/create-template.sh b/ci-cd automation/helper-scripts/create-template.sh similarity index 85% rename from container maintenance/helper-scripts/create-template.sh rename to ci-cd automation/helper-scripts/create-template.sh index 2f80b706..54d5e1ea 100755 --- a/container maintenance/helper-scripts/create-template.sh +++ b/ci-cd automation/helper-scripts/create-template.sh @@ -41,8 +41,8 @@ else --full true fi -AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/registration-token --header "Authorization: token $GITHUB_PAT") -TOKEN=$(echo "$AUTH_TOKEN_RESPONSE" | jq -r '.token') +# AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/registration-token --header "Authorization: token $GITHUB_PAT") +# TOKEN=$(echo "$AUTH_TOKEN_RESPONSE" | jq -r '.token') # Remove rsa keys ==== pct start $NEXT_ID diff --git a/container maintenance/helper-scripts/delete-runner.sh b/ci-cd automation/helper-scripts/delete-runner.sh old mode 100644 new mode 100755 similarity index 97% rename from container maintenance/helper-scripts/delete-runner.sh rename to ci-cd automation/helper-scripts/delete-runner.sh index dd88b530..e1d2241f --- a/container maintenance/helper-scripts/delete-runner.sh +++ b/ci-cd automation/helper-scripts/delete-runner.sh @@ -7,11 +7,13 @@ PROXMOX_USERNAME="$3" PROXMOX_PASSWORD="$4" CONTAINER_NAME="$5" +set -x sleep 2 # Delete Container CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$CONTAINER_NAME" '$3 == name {print $1}') +echo "echo $CONTAINER_ID" if (( $CONTAINER_ID % 2 == 0 )); then if ssh root@10.15.0.5 "pct status $CONTAINER_ID" | grep -q "status: running"; then diff --git a/container maintenance/helper-scripts/repository_status.sh b/ci-cd automation/helper-scripts/repository_status.sh old mode 100644 new mode 100755 similarity index 100% rename from container maintenance/helper-scripts/repository_status.sh rename to ci-cd automation/helper-scripts/repository_status.sh diff --git a/container maintenance/helper-scripts/verify_container_ownership.sh b/ci-cd automation/helper-scripts/verify_container_ownership.sh old mode 100644 new mode 100755 similarity index 87% rename from container maintenance/helper-scripts/verify_container_ownership.sh rename to ci-cd automation/helper-scripts/verify_container_ownership.sh index 53b9e1cd..a0727cc2 --- a/container maintenance/helper-scripts/verify_container_ownership.sh +++ b/ci-cd automation/helper-scripts/verify_container_ownership.sh @@ -20,18 +20,18 @@ if [ "$TYPE_RUNNER" != "true" ]; then if (( $CONTAINER_ID % 2 == 0 )); then CONTAINER_OWNERSHIP=$(ssh root@10.15.0.5 "pct config \"$CONTAINER_ID\" | grep "tags" | grep \"$PROXMOX_USERNAME\"") else - CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep -x "tags: $PROXMOX_USERNAME") + CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep -E "(^|;)$PROXMOX_USERNAME(;|$)") fi else CONTAINER_OWNERSHIP=$(ssh root@10.15.0.5 "pct config \"$CONTAINER_ID\" | grep "tags" | grep \"$PROXMOX_USERNAME\"") PVE1="false" if [ -z "$CONTAINER_OWNERSHIP" ]; then - CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep -x "tags: $PROXMOX_USERNAME") + CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep -E "(^|;)$PROXMOX_USERNAME(;|$)") PVE1="true" fi fi if [ -z "$CONTAINER_OWNERSHIP" ]; then echo "❌ You do not own the container with name \"$CONTAINER_NAME\"." - return 2 + outputError 1 "You do not own the container with name \"$CONTAINER_NAME\"." fi diff --git a/ci-cd automation/proxmox-launchpad/.gitignore b/ci-cd automation/proxmox-launchpad/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/ci-cd automation/proxmox-launchpad/README.md b/ci-cd automation/proxmox-launchpad/README.md new file mode 100644 index 00000000..17c2a011 --- /dev/null +++ b/ci-cd automation/proxmox-launchpad/README.md @@ -0,0 +1,308 @@ +# Proxmox LaunchPad + +This GitHub action utilizes MIE's open source cluster to manage LXC containers derived from your github repository source code. + +> [!NOTE] +> This project is new and is in a early version. There are likely bugs. If you encounter any, please create an issue. + +## Table of Contents +1. [Video Walkthroughs](#video-walkthroughs) +2. [Sequence Diagram](#sequence-diagram) +3. [Prerequisites](#prerequisites) +4. [Getting Started](#getting-started) + - [Create-Runner Job](#create-runner-workflow-job) + - [Personal Access Token](#creating-a-github-pat-for-your-workflow) + - [Runner Job](#runner-job) + - [Manage-Container Job](#manage-container-workflow-job) +5. [Configurations](#configurations) + - [Basic Properties](#basic-properties) + - [Automatic Deployment Properties](#automatic-deployment-properties) +6. [Important Notes for Automatic Deployment](#important-notes-for-automatic-deployment) +7. [Output](#output) +8. [Sample Workflow File ](#sample-workflow-file) +9. [Misc.](#misc) + +## Video Walkthroughs + +I have created a series of videos to walk you through automatic deployment, both in GitHub and via the command line. + +**[Long-Form]** Proxmox LaunchPad Walkthrough: [Video](https://youtu.be/Xa2L1o-atEM)
+**[Short-Form]** Proxmox LaunchPad Demonstration: [Short](https://youtube.com/shorts/SuK73Jej5j4)
+**[Long-Form]** Automatic Deployment through Command Line: [Video](https://youtu.be/acDW-a32Yr8)
+**[Long-Form]** Getting Started with Creating LXC Continers with Proxmox: [Video](https://youtu.be/sVW3dkBqs4E) + +## Sequence Diagram + +The sequence diagram below describes the sequence of events executed by this Github Action. + +```mermaid +sequenceDiagram + participant Dev as Developer + participant GH as GitHub + participant GHAR as GitHub Actions Runner (hosted) + participant Prox as Proxmox Cluster + participant LXC as LXC Container (Self-hosted Runner) + + Dev->>GH: Push/Create/Delete branch + GH->>GHAR: Trigger workflow + + alt Push/Create event + GHAR->>Prox: Check if LXC container exists for branch + alt Container does not exist + GHAR->>Prox: Clone template, create LXC container + Prox->>LXC: Start container, configure self-hosted runner + GHAR->>LXC: Register self-hosted runner + GHAR->>LXC: Run manage container job (install deps, clone repo, install services, deploy app) + else Container exists + GHAR->>Prox: Call update script + Prox->>LXC: Update container contents, restart with latest branch + end + else Delete event + GHAR->>LXC: Call delete-container script + LXC->>Prox: Remove runner and delete LXC container + end +``` + +## Prerequisites +- Proxmox Datacluster Setup that mirrors/forks [https://github.com/mieweb/opensource-server](https://github.com/mieweb/opensource-server). +- Valid Proxmox Account. + +## Getting Started + +> [!WARNING] +> This Github Action requires you to pass your Github Personal Access Token in order to create runners. If you are comfortable doing this, see [Create-Runner Job](#create-runner-workflow-job). If you are not, you may supply your own self-hosted runner and skip to [Manage-Container Job](#manage-container-workflow-job). + +To use this action in your repository, you need to add the following trigger events in a workflow file: + +```yaml +on: + push: + create: + delete: +``` + +This allows a container to be created/updated on a push command, created when a new branch is created, and deleted when a branch is deleted (like in the case of an accepted PR). + +### Create-Runner Workflow Job + +> [!CAUTION] +> If you choose to pass in your GitHub Personal Access Token, keep it in a secure place and do not share it with anyone. + +#### Creating a GitHub PAT for your Workflow + +This Github Action requires you to pass your Github Personal Access Token in order to create runners. To create a PAT, navigate to your GitHub account settings. Then, on the bottom left-hand side, click developer settings. Navigate to Personal Access Tokens (classic). Click on generate new token, then give your token a name and an expiration date. Finally, select the manage_runners:org permission or the manage_runners:enterprise permission, depending on where your repository is housed. Finally, a token should be generated. Make sure to place the token somewhere securely. Then, add it as a repository secret in the repository that you want to run your workflow file in. + +#### Runner Job + +Before a container can be managed, a self-hosted runner must be installed on the LXC container to complete future workflow jobs. To do this, a github-supplied runner needs to create the container and install/start a custom runner on it that is linked to your repository. + +The create-runner job in your workflow file should look similar to this: + +```yaml +setup-runner: + runs-on: ubuntu-latest + steps: + - name: Install Dependencies + run: | + sudo apt install -y sshpass jq + + - uses: maxklema/proxmox-launchpad@main + with: + proxmox_password: ${{ secrets.PROXMOX_PASSWORD }} + proxmox_username: ${{ secrets.PROXMOX_USERNAME }} + github_pat: ${{ secrets.GH_PAT }} +``` + +The GitHub runner needs to install sshpass (used to authenticate into another host using password authentication) and jq (a popular package for managing/parsing JSON data). + +In the second step, 3 fields are required: `proxmox_username`, `proxmox_password`, and `github_pat` + +To see an explanation for these fields: See [Basic Properties](#basic-properties) + + +### Manage-Container Workflow Job + +The second job in your workflow file should look similar to this: + +> [!NOTE] +> If you chose to run this on your own self-hosted runner instead of the action creating one for you, this will be your first job. Therefore, the needs parameter is not needed. + +```yaml + manage-container: + runs-on: self-hosted + needs: setup-runner + steps: + - uses: maxklema/proxmox-launchpad@test + with: + proxmox_password: ${{ secrets.PROXMOX_PASSWORD }} + proxmox_username: ${{ secrets.PROXMOX_USERNAME }} +``` + + + +## Configurations + +At the very minimum, two configuration settings are required to create any container. With all of these properties specified, you can create an empty container for a branch. + +### Basic Properties + +| Propety | Required? | Description | Supplied by Github? | +| ---------------- | ------ | ---------------------------------------------- | ------ | +| `proxmox_username` | Yes | Your proxmox username assigned to you. | N/A +| `proxmox_password` | Yes | Your proxmox password assigned to you. | N/A +| `http_port` | No | The HTTP Port for your container to listen on. It must be between `80` and `60000`. Default value is `3000`. | N/A +| `linux_distribution` | No | The Linux Distribution that runs on your container. Currently, `rocky` (Rocky 9.5) and `debian` (Debian 12) are available. Default value is `Debian`. | N/A +| `github_pat` | Conditional | Your GitHub Personal Access Token. This is used to manage runners in your containers. This is **only required if you want the workflow to create runners for you.**| Yes. Accessable in developer settings. | + + +There are a few other properties that are not required, but can still be specified in the workflow file: +
+ +| Propety | Required? | Description | Supplied by Github? | +| --------- | ----- | ------------------------------------ | ------ | +| `public_key` | No | Your machine's public key that will be stored in the `~/.ssh/authorized_keys` file of your repository. This allows you to SSH into your container without a password. It is more secure and recommended. | N/A + +### Automatic Deployment Properties + +This github action can *attempt* to automatically deploy services on your container. This is done by fetching your repository contents on the branch that the script is being ran in, installing dependencies/services, and running build and start commands in the background. + +Additionally, with automatic deployment enabled, your container will update on every push command automatically, preventing you from having to SSH into the container and setting it up manually. + +> [!NOTE] +> Properties below that are required assuming you want to automatically deploy your project. If not, none of these properties are needed. + +| Propety | Required? | Description | +| --------- | ----- | ------------------------------------ | +| `project_root` | No | The root directory of your project to deploy from. Example: `/flask-server`. If the root directory is the same as the github root directory, leave blank. +| `services` | No | A JSON array of services to add to your container. Example: ```services: '["mongodb", "docker"]'```. These services will automatically install and start up on container creation. **NOTE**: All services in this list must belong on the list of available services below. If you need a service that is not on the list, see `custom_services`.

Available Services: `meteor`, `mongodb`, `docker`, `redis`, `postgresql`, `apache`, `nginx`, `rabbitmq`, `memcached`, `mariadb`. +| `custom_services` | No | A 2D JSON array of custom service installation commands to install any custom service(s) not in `services`.

Example: ```custom_services: [["sudo apt-get install -y service", "sudo systemctl enable service", "sudo systemctl start service"], ["sudo apt-get install -y service2", "sudo systemctl enable service2", "sudo systemctl start service2"]]``` + + +There are two types of deployments: single component and multi-component deployment. Single component deployment involves deploying only a single service (i.e. a single Flask Server, REACT application, MCP Server, etc.). Multi-component deployment involves deploying more than one service at the same time (i.e. a flask backend and a vite.js backend). + +> [!IMPORTANT] +> In Multi-Component applications, each top-layer key represents the file path, relative to the root directory, to the component (service) to place those variables/commands in. + +| Propety | Required? | Description | Single Component | Multi-Component | +| --------- | ----- | ------------------------------------ | ---- | --- | +| `container_env_vars` | No. | Key-Value Environment variable pairs. | Dictionary in the form of: `{ "api_key": "123", "password": "abc"}` | Dictionary in the form of: `'{"/frontend": { "api_key": "123"}, "/backend": { "password": "abc123" }}'`. +| `install_command` | Yes | Commands to install all project dependencies | String of the installation command, i.e. `npm install`. | Dictionary in the form of: `'{"/frontend": "npm install", "/backend": "pip install -r ../requirements.txt"}'`. +| `build_command` | No | Commands to build project components | String of the build command, i.e. `npm build`. | Dictionary in the form of: `'{"/frontend": "npm build", "/backend": "python3 build.py"}'`. +| `start_command` | Yes | Commands to start project components. | String of the start command, i.e. `npm run`. | Dictionary in the form of: `'{"/frontend": "npm run", "/backend": "flask run"}'`. +| `runtime_language` | Yes | Runtime language of each project component, which can either be `nodejs` or `python`. | String of runtime environment, i.e. `nodejs` | Dictionary in the form of: `'{"/frontend": "nodejs", "/backend": "python"}'`. +| `root_start_command` | No | Command to run at the project directory root for **multi-component applications**. | N/A | String of the command, i.e. `Docker run` + +## Important Notes for Automatic Deployment + +Below are some important things to keep in mind if you want your application to be automatically deployed: +- If you are using meteor, you must start your application with the flags ``--allow-superuser`` and `--port 0.0.0.0:`. + - Meteor is a large package, so deploying it may take more time than other applications. +- When running a service, ensure it is listening on `0.0.0.0` (your IP) instead of only locally at `127.0.0.1`. +- The Github action will fail with an exit code and message if a property is not set up correctly. + + +## Output + +When a container is successfully created (Github Action is successful), you will see an output with all of your container details. This includes all your ports, container ID, container IP Address (internal in 10.15.x.x subnet), public domain name, and ssh command to access your container. + +See an example output below: + +``` +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +🔔 COPY THESE PORTS DOWN — For External Access +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +📌 Note: Your container listens on SSH Port 22 internally, + but EXTERNAL traffic must use the SSH port listed below: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +✅ Hostname Registration: polyglot-test-maxklema-pull-request → 10.15.129.23 +🔐 SSH Port : 2344 +🌐 HTTP Port : 32000 +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +📦 Container ID : 136 +🌐 Internal IP : 10.15.129.23 +🔗 Domain Name : https://polyglot-test-maxklema-pull-request.opensource.mieweb.org +🛠️ SSH Access : ssh -p 2344 root@polyglot-test-maxklema-pull-request.opensource.mieweb.org +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +NOTE: Additional background scripts are being ran in detached terminal sessions. +Wait up to two minutes for all processes to complete. +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Still not working? Contact Max K. at maxklema@gmail.com +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +``` + +> [!NOTE] +> Even if your GitHub Action workflow is finished, *it may not be accessible right away. Background tasks (migration, template cloning, cleanup, etc) are still be ran in detatched terminal sessions*. Wait a few minutes for all tasks to complete. + +## Sample Workflow File + +The workflow file below is an example workflow designed to deploy a multi-component application with a python (flask) backend and nodejs (vite) frontend: + +**With PAT:** + +```yaml +name: Proxmox Container Management + +on: + push: + create: + delete: + +jobs: + setup-runner: + runs-on: ubuntu-latest + steps: + - name: Install Dependencies + run: | + sudo apt install -y sshpass jq + - uses: maxklema/proxmox-launchpad@test + with: + proxmox_password: ${{ secrets.PROXMOX_PASSWORD }} + proxmox_username: ${{ secrets.PROXMOX_USERNAME }} + github_pat: ${{ secrets.GH_PAT }} + manage-container: + runs-on: self-hosted + needs: setup-runner + steps: + - uses: maxklema/proxmox-launchpad@test + with: + proxmox_password: ${{ secrets.PROXMOX_PASSWORD }} + proxmox_username: ${{ secrets.PROXMOX_USERNAME }} + public_key: ${{ secrets.PUBLIC_KEY }} + container_env_vars: '{"API_KEY": "1234"}' + install_command: npm i + start_command: npm start + runtime_language: nodejs + services: '["mongodb"]' +``` + +**Without PAT:** + +```yaml +name: Proxmox Container Management + +on: + push: + create: + delete: + +jobs: + manage-container: + runs-on: self-hosted + needs: setup-runner + steps: + - uses: maxklema/proxmox-launchpad@test + with: + proxmox_password: ${{ secrets.PROXMOX_PASSWORD }} + proxmox_username: ${{ secrets.PROXMOX_USERNAME }} + public_key: ${{ secrets.PUBLIC_KEY }} + container_env_vars: '{"API_KEY": "1234"}' + install_command: npm i + start_command: npm start + runtime_language: nodejs + services: '["mongodb"]' +``` + + +## Misc. +Feel free to submit a PR/issue here or in [opensource-server](https://github.com/mieweb/opensource-server). +Author: [@maxklema](https://github.com/maxklema) diff --git a/ci-cd automation/proxmox-launchpad/action.yml b/ci-cd automation/proxmox-launchpad/action.yml new file mode 100644 index 00000000..79f26b59 --- /dev/null +++ b/ci-cd automation/proxmox-launchpad/action.yml @@ -0,0 +1,494 @@ +# action.yml +name: Proxmox LaunchPad +description: Manage Proxmox Containers for your Repository. +author: maxklema +branding: + icon: "package" + color: "purple" + +inputs: + proxmox_username: + required: true + proxmox_password: + required: true + container_password: + required: false + public_key: + required: false + http_port: + required: false + project_root: + required: false + container_env_vars: + required: false + install_command: + required: false + build_command: + required: false + start_command: + required: false + runtime_language: + required: false + services: + required: false + custom_services: + required: false + linux_distribution: + required: false + multi_component: + required: false + root_start_command: + required: false + github_pat: + required: false + +runs: + using: "composite" + steps: + - name: Check if action should run + shell: bash + id: should-run + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_EVENT_CREATED: ${{ github.event.created }} + run: | + if [[ "$GITHUB_EVENT_NAME" != "push" ]] || [[ "$GITHUB_EVENT_CREATED" == "false" ]]; then + echo "should_run=true" >> $GITHUB_OUTPUT + else + echo "should_run=false" >> $GITHUB_OUTPUT + echo "Skipping action: Push event with created=true" + fi + + - name: Determine Target Branch Name + shell: bash + id: branch-name + if: steps.should-run.outputs.should_run == 'true' + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_EVENT_REF: ${{ github.event.ref }} + run: | + if [[ "$GITHUB_EVENT_NAME" == "delete" ]]; then + TARGET_BRANCH="$GITHUB_EVENT_REF" + echo "Using deleted branch name: $TARGET_BRANCH" + else + TARGET_BRANCH="$GITHUB_REF_NAME" + echo "Using current branch name: $TARGET_BRANCH" + fi + echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT + + - name: Create Runner (If Needed) + shell: bash + id: create-runner + if: steps.should-run.outputs.should_run == 'true' + env: + GITHUB_REPOSITORY_FULL: ${{ github.repository }} + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + TARGET_BRANCH: ${{ steps.branch-name.outputs.target_branch }} + CONTAINER_PASSWORD: ${{ inputs.container_password }} + PROXMOX_USERNAME: ${{ inputs.proxmox_username }} + PROXMOX_PASSWORD: ${{ inputs.proxmox_password }} + GITHUB_PAT: ${{ inputs.github_pat }} + GITHUB_API: ${{ github.api_url }} + LINUX_DISTRIBUTION: ${{ inputs.linux_distribution }} + PROJECT_REPOSITORY: ${{ github.server_url }}/${{ github.repository }} + GITHUB_JOB: ${{ github.job }} + run: | + REPO_NAME=$(basename "$GITHUB_REPOSITORY_FULL") + CONTAINER_NAME="${GITHUB_REPOSITORY_OWNER}-${REPO_NAME}-${TARGET_BRANCH}" + CONTAINER_NAME=${CONTAINER_NAME,,} + CONTAINER_NAME=$(echo "$CONTAINER_NAME" | sed 's/[^a-z0-9-]/-/g') + export CONTAINER_NAME + + # Auto-detect if this is a runner setup job based on job name or if no container inputs are provided + CREATE_RUNNER_JOB="N" + if [[ "$GITHUB_JOB" == *"setup"* ]] || [[ "$GITHUB_JOB" == *"runner"* ]]; then + CREATE_RUNNER_JOB="Y" + echo "CREATE_RUNNER_JOB=true" >> $GITHUB_OUTPUT + fi + + if [ ! -z "$GITHUB_PAT" ]; then + RESPONSE=$(curl --location ${GITHUB_API}/repos/${GITHUB_REPOSITORY_OWNER}/${REPO_NAME}/actions/runners --header "Authorization: token $GITHUB_PAT") + + while read -r RUN; do + RUNNER_NAME=$(echo "$RUN" | jq -r '.name') + if [ "$RUNNER_NAME" == "$CONTAINER_NAME" ]; then + if [ "${CREATE_RUNNER_JOB^^}" == "N" ]; then + exit 0 #Runner exists, continue to next steps + else + echo "STOP_SCRIPT=true" >> $GITHUB_OUTPUT + exit 0 # Runner exists, continue to next job. + fi + fi + done < <(echo "$RESPONSE" | jq -c '.runners[]') + + echo "Creating a Runner..." + set +e + sshpass -p 'mie123!' ssh \ + -T \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o SendEnv="CONTAINER_NAME CONTAINER_PASSWORD PROXMOX_USERNAME PROXMOX_PASSWORD GITHUB_PAT LINUX_DISTRIBUTION PROJECT_REPOSITORY" \ + setup-runner@opensource.mieweb.org + + EXIT_STATUS=$? + + # Exit if a container exists but an associated runner does not. + if [ $EXIT_STATUS != 3 ]; then + echo "Something went wrong with creating/using a runner." + exit 1 + fi + + echo "STOP_SCRIPT=true" >> $GITHUB_OUTPUT + fi + + - name: Container Creation for Branch (If Needed) + id: create-lxc + shell: bash + if: ${{ (github.event_name == 'create' || github.event_name == 'push') && steps.should-run.outputs.should_run == 'true' }} + env: + GITHUB_EVENT: ${{ github.event_name }} + GITHUB_REPOSITORY_FULL: ${{ github.repository }} + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + TARGET_BRANCH: ${{ steps.branch-name.outputs.target_branch }} + CONTAINER_PASSWORD: ${{ inputs.container_password }} + PROXMOX_USERNAME: ${{ inputs.proxmox_username }} + PROXMOX_PASSWORD: ${{ inputs.proxmox_password }} + PUBLIC_KEY: ${{ inputs.public_key }} + HTTP_PORT: ${{ inputs.http_port }} + DEPLOY_ON_START: ${{ inputs.deploy_on_start }} + PROJECT_REPOSITORY: ${{ github.server_url }}/${{ github.repository }} + PROJECT_BRANCH: ${{ steps.branch-name.outputs.target_branch }} + PROJECT_ROOT: ${{ inputs.project_root }} + REQUIRE_ENV_VARS: ${{ inputs.require_env_vars }} + CONTAINER_ENV_VARS: ${{ inputs.container_env_vars }} + INSTALL_COMMAND: ${{ inputs.install_command }} + START_COMMAND: ${{ inputs.start_command }} + BUILD_COMMAND: ${{ inputs.build_command }} + RUNTIME_LANGUAGE: ${{ inputs.runtime_language }} + REQUIRE_SERVICES: ${{ inputs.require_services }} + SERVICES: ${{ inputs.services }} + CUSTOM_SERVICES: ${{ inputs.custom_services }} + LINUX_DISTRIBUTION: ${{ inputs.linux_distribution }} + MULTI_COMPONENT: ${{ inputs.multi_component }} + ROOT_START_COMMAND: ${{ inputs.root_start_command }} + GITHUB_PAT: ${{ inputs.github_pat }} + GH_ACTION: y + run: | + set +e + REPO_NAME=$(basename "$GITHUB_REPOSITORY_FULL") + CONTAINER_NAME="${GITHUB_REPOSITORY_OWNER}-${REPO_NAME}-${TARGET_BRANCH}" + CONTAINER_NAME=${CONTAINER_NAME,,} + CONTAINER_NAME=$(echo "$CONTAINER_NAME" | sed 's/[^a-z0-9-]/-/g') + export CONTAINER_NAME + STOP_SCRIPT=${{ steps.create-runner.outputs.STOP_SCRIPT }} + if [ "$STOP_SCRIPT" != "true" ]; then + set +e + echo "Running Container Exists..." + + # Determine SSH target based on network location + EXTERNAL_IP=$(dig +short opensource.mieweb.org) + if [ "$EXTERNAL_IP" = "10.15.20.69" ]; then + SSH_TARGET="10.15.0.4" + else + SSH_TARGET="opensource.mieweb.org" + fi + + sshpass -p 'mie123!' ssh \ + -T \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o SendEnv="PROXMOX_USERNAME PROXMOX_PASSWORD CONTAINER_NAME PROJECT_REPOSITORY" \ + container-exists@$SSH_TARGET + CONTAINER_EXISTS=$? + if [ $CONTAINER_EXISTS -eq 1 ]; then + echo "FAILED=1" >> $GITHUB_ENV # User does not own the container + elif [ $CONTAINER_EXISTS -eq 0 ]; then + echo "Cloning repository based on $PROJECT_BRANCH branch." + + sshpass -p 'mie123!' ssh \ + -T \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o SendEnv="CONTAINER_NAME CONTAINER_PASSWORD PROXMOX_USERNAME PUBLIC_KEY PROXMOX_PASSWORD HTTP_PORT DEPLOY_ON_START PROJECT_REPOSITORY PROJECT_BRANCH PROJECT_ROOT REQUIRE_ENV_VARS CONTAINER_ENV_VARS INSTALL_COMMAND START_COMMAND RUNTIME_LANGUAGE REQUIRE_SERVICES SERVICES CUSTOM_SERVICES LINUX_DISTRIBUTION MULTI_COMPONENT ROOT_START_COMMAND GH_ACTION GITHUB_PAT" \ + create-container@$SSH_TARGET + + CONTAINER_CREATED=$? + echo "CONTAINER_CREATED=true" >> $GITHUB_OUTPUT + if [ $CONTAINER_CREATED -ne 0 ]; then + echo "FAILED=1" >> $GITHUB_ENV + fi + fi + fi + + - name: Container Update on Branch Push + shell: bash + if: ${{ (github.event_name == 'push' && steps.create-lxc.outputs.CONTAINER_CREATED != 'true') && steps.should-run.outputs.should_run == 'true' }} + env: + GITHUB_EVENT: ${{ github.event_name }} + GITHUB_REPOSITORY_FULL: ${{ github.repository }} + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + TARGET_BRANCH: ${{ steps.branch-name.outputs.target_branch }} + PROXMOX_USERNAME: ${{ inputs.proxmox_username }} + PROXMOX_PASSWORD: ${{ inputs.proxmox_password }} + PROJECT_REPOSITORY: ${{ github.server_url }}/${{ github.repository }} + PROJECT_BRANCH: ${{ steps.branch-name.outputs.target_branch }} + PROJECT_ROOT: ${{ inputs.project_root }} + INSTALL_COMMAND: ${{ inputs.install_command }} + START_COMMAND: ${{ inputs.start_command }} + BUILD_COMMAND: ${{ inputs.build_command }} + RUNTIME_LANGUAGE: ${{ inputs.runtime_language }} + MULTI_COMPONENT: ${{ inputs.multi_component }} + SERVICES: ${{ inputs.services }} + CUSTOM_SERVICES: ${{ inputs.custom_services }} + REQUIRE_SERVICES: ${{ inputs.require_services }} + LINUX_DISTRIBUTION: ${{ inputs.linux_distribution }} + DEPLOY_ON_START: ${{ inputs.deploy_on_start }} + ROOT_START_COMMAND: ${{ inputs.root_start_command }} + GITHUB_PAT: ${{ inputs.github_pat }} + HTTP_PORT: ${{ inputs.http_port }} + GH_ACTION: y + run: | + set +e + echo "Running Container Update..." + REPO_NAME=$(basename "$GITHUB_REPOSITORY_FULL") + CONTAINER_NAME="${GITHUB_REPOSITORY_OWNER}-${REPO_NAME}-${TARGET_BRANCH}" + CONTAINER_NAME=${CONTAINER_NAME,,} + CONTAINER_NAME=$(echo "$CONTAINER_NAME" | sed 's/[^a-z0-9-]/-/g') + export CONTAINER_NAME + echo "$LINUX_DISTRIBUTION" + STOP_SCRIPT=${{ steps.create-runner.outputs.STOP_SCRIPT }} + if [ "$STOP_SCRIPT" != true ]; then + # Determine SSH target based on network location + EXTERNAL_IP=$(dig +short opensource.mieweb.org) + if [ "$EXTERNAL_IP" = "10.15.20.69" ]; then + SSH_TARGET="10.15.0.4" + else + SSH_TARGET="opensource.mieweb.org" + fi + + sshpass -p 'mie123!' ssh \ + -T \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o SendEnv="CONTAINER_NAME PROXMOX_USERNAME PROXMOX_PASSWORD PROJECT_REPOSITORY PROJECT_BRANCH PROJECT_ROOT INSTALL_COMMAND START_COMMAND BUILD_COMMAND RUNTIME_LANGUAGE MULTI_COMPONENT ROOT_START_COMMAND DEPLOY_ON_START SERVICES CUSTOM_SERVICES REQUIRE_SERVICES LINUX_DISTRIBUTION GH_ACTION HTTP_PORT" \ + update-container@$SSH_TARGET + UPDATE_EXIT=$? + if [ $UPDATE_EXIT -ne 0 ]; then + echo "FAILED=1" >> $GITHUB_ENV + fi + fi + + - name: Container Deletion on Branch Deletion (Check) + shell: bash + if: ${{ github.event_name == 'delete' && steps.should-run.outputs.should_run == 'true' }} + env: + GITHUB_EVENT: ${{ github.event_name }} + GITHUB_REPOSITORY_FULL: ${{ github.repository }} + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + TARGET_BRANCH: ${{ steps.branch-name.outputs.target_branch }} + PROXMOX_USERNAME: ${{ inputs.proxmox_username }} + PROXMOX_PASSWORD: ${{ inputs.proxmox_password }} + PROJECT_REPOSITORY: ${{ github.server_url }}/${{ github.repository }} + GITHUB_PAT: ${{ inputs.github_pat }} + run: | + set +e + REPO_NAME=$(basename "$GITHUB_REPOSITORY_FULL") + CONTAINER_NAME="${GITHUB_REPOSITORY_OWNER}-${REPO_NAME}-${TARGET_BRANCH}" + CONTAINER_NAME=${CONTAINER_NAME,,} + CONTAINER_NAME=$(echo "$CONTAINER_NAME" | sed 's/[^a-z0-9-]/-/g') + export CONTAINER_NAME + STOP_SCRIPT=${{ steps.create-runner.outputs.STOP_SCRIPT }} + if [ "$STOP_SCRIPT" != true ]; then + # Determine SSH target based on network location + EXTERNAL_IP=$(dig +short opensource.mieweb.org) + if [ "$EXTERNAL_IP" = "10.15.20.69" ]; then + SSH_TARGET="10.15.0.4" + else + SSH_TARGET="opensource.mieweb.org" + fi + + sshpass -p 'mie123!' ssh \ + -T \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o SendEnv="PROXMOX_USERNAME PROXMOX_PASSWORD CONTAINER_NAME GITHUB_PAT PROJECT_REPOSITORY" \ + delete-container@$SSH_TARGET + DELETE_EXIT=$? + if [ $DELETE_EXIT -ne 0 ]; then + echo "FAILED=1" >> $GITHUB_ENV + fi + fi + + - name: Check if branch is part of a PR and comment + shell: bash + id: check-pr + if: steps.should-run.outputs.should_run == 'true' && steps.create-runner.outputs.CREATE_RUNNER_JOB != 'true' && env.FAILED != '1' + env: + GITHUB_TOKEN: ${{ inputs.github_pat }} + GITHUB_REPOSITORY: ${{ github.repository }} + TARGET_BRANCH: ${{ steps.branch-name.outputs.target_branch }} + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + RUN_ID: ${{ github.run_id }} + run: | + if [ -z "$GITHUB_TOKEN" ]; then + echo "pr_number=" >> $GITHUB_OUTPUT + echo "is_pr=false" >> $GITHUB_OUTPUT + echo "No GitHub token provided, skipping PR detection" + exit 0 + fi + + # Check if this branch has an open PR + PR_DATA=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/pulls?state=open&head=${{ github.repository_owner }}:$TARGET_BRANCH") + + PR_NUMBER=$(echo "$PR_DATA" | jq -r '.[0].number // empty') + + if [ -n "$PR_NUMBER" ] && [ "$PR_NUMBER" != "null" ]; then + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + echo "is_pr=true" >> $GITHUB_OUTPUT + echo "Branch $TARGET_BRANCH is part of PR #$PR_NUMBER" + + # Generate container name + REPO_NAME=$(basename "$GITHUB_REPOSITORY") + CONTAINER_NAME="${GITHUB_REPOSITORY_OWNER}-${REPO_NAME}-${TARGET_BRANCH}" + CONTAINER_NAME=${CONTAINER_NAME,,} + CONTAINER_NAME=$(echo "$CONTAINER_NAME" | sed 's/[^a-z0-9-]/-/g') + + # Create initial comment on PR + CONTAINER_URL="https://${CONTAINER_NAME}.opensource.mieweb.org" + + COMMENT_BODY="## 🚀 Proxmox LaunchPad Action + **Expected URL**: [$CONTAINER_NAME]($CONTAINER_URL) *(will be available once deployment completes)* + **Status**: ✅ Application was deployed according to workflow configurations. + **Branch**: \`$TARGET_BRANCH\` + **Run ID**: [\`$RUN_ID\`](https://github.com/$GITHUB_REPOSITORY/actions/runs/$RUN_ID) + **Container Name**: \`$CONTAINER_NAME\` + + > This comment was automatically generated by Proxmox LaunchPad: The fastest way to deploy your repository code. To use Proxmox in your own repository, see: [Proxmox LaunchPad](https://github.com/marketplace/actions/proxmox-launchpad)." + + # Use jq to safely build the JSON payload from the variable + JSON_PAYLOAD=$(jq -n --arg body "$COMMENT_BODY" '{body: $body}') + + # Post the initial comment + curl -s -X POST \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "Content-Type: application/json" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + -d "$JSON_PAYLOAD" > /dev/null + + echo "Initial comment posted to PR #$PR_NUMBER" + else + echo "pr_number=" >> $GITHUB_OUTPUT + echo "is_pr=false" >> $GITHUB_OUTPUT + echo "Branch $TARGET_BRANCH is not part of any open PR" + fi + + - name: Comment on PR on Failure + if: env.FAILED == '1' + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.github_pat }} + GITHUB_REPOSITORY: ${{ github.repository }} + TARGET_BRANCH: ${{ steps.branch-name.outputs.target_branch }} + RUN_ID: ${{ github.run_id }} + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + run: | + if [ -z "$GITHUB_TOKEN" ]; then + echo "Cannot comment on PR: missing token" + exit 1 + fi + + # Check if this branch has an open PR + PR_DATA=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/pulls?state=open&head=${GITHUB_REPOSITORY_OWNER}:$TARGET_BRANCH") + + PR_NUMBER=$(echo "$PR_DATA" | jq -r '.[0].number // empty') + + if [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" == "null" ]; then + echo "Not a pull request, skipping failure comment." + exit 0 + fi + + REPO_NAME=$(basename "$GITHUB_REPOSITORY") + CONTAINER_NAME="${GITHUB_REPOSITORY_OWNER}-${REPO_NAME}-${TARGET_BRANCH}" + CONTAINER_NAME=${CONTAINER_NAME,,} + CONTAINER_NAME=$(echo "$CONTAINER_NAME" | sed 's/[^a-z0-9-]/-/g') + + CONTAINER_URL="https://${CONTAINER_NAME}.opensource.mieweb.org" + + COMMENT_BODY="## 🚀 Proxmox LaunchPad Action + **Expected URL**: [$CONTAINER_NAME]($CONTAINER_URL) *(will be available once deployment completes)* + **Status**: ❌ Application failed to deploy. View [\`$RUN_ID\`](https://github.com/$GITHUB_REPOSITORY/actions/runs/$RUN_ID) to see logs. + **Branch**: \`$TARGET_BRANCH\` + **Run ID**: [\`$RUN_ID\`](https://github.com/$GITHUB_REPOSITORY/actions/runs/$RUN_ID) + **Container Name**: \`$CONTAINER_NAME\` + + > This comment was automatically generated by Proxmox LaunchPad: The fastest way to deploy your repository code. To use Proxmox in your own repository, see: [Proxmox LaunchPad](https://github.com/marketplace/actions/proxmox-launchpad)." + + JSON_PAYLOAD=$(jq -n --arg body "$COMMENT_BODY" '{body: $body}') + + # Post the comment + curl -s -X POST \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "Content-Type: application/json" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + -d "$JSON_PAYLOAD" > /dev/null + + echo "Failure comment posted to PR #$PR_NUMBER" + exit 1 + + - name: Create GitHub Deployment (Default Branch) + if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && env.FAILED != '1' + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.github_pat }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_REF: ${{ github.ref }} + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + run: | + REPO_NAME=$(basename "$GITHUB_REPOSITORY") + CONTAINER_NAME="${GITHUB_REPOSITORY_OWNER}-${REPO_NAME}-${GITHUB_REF#refs/heads/}" + CONTAINER_NAME=${CONTAINER_NAME,,} + CONTAINER_NAME=$(echo "$CONTAINER_NAME" | sed 's/[^a-z0-9-]/-/g') + CONTAINER_URL="https://${CONTAINER_NAME}.opensource.mieweb.org" + DEPLOYMENT_RESPONSE=$(curl -s -X POST \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "Content-Type: application/json" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/deployments" \ + -d '{ + "ref": "'${GITHUB_REF#refs/heads/}'", + "required_contexts": [], + "environment": "Preview - '$GITHUB_REPOSITORY'", + "description": "Deployment triggered from Proxmox LaunchPad action.", + "sha": "'$GITHUB_SHA'" + }') + DEPLOYMENT_ID=$(echo "$DEPLOYMENT_RESPONSE" | jq -r '.id') + if [ "$DEPLOYMENT_ID" != "null" ] && [ -n "$DEPLOYMENT_ID" ]; then + curl -s -X POST \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "Content-Type: application/json" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/deployments/$DEPLOYMENT_ID/statuses" \ + -d '{ + "state": "success", + "description": "Deployment completed successfully.", + "environment": "Preview - '$GITHUB_REPOSITORY'", + "environment_url": "'$CONTAINER_URL'" + }' > /dev/null + echo "Deployment created and marked as successful for default branch: ${GITHUB_REF#refs/heads/}" + echo "Deployment URL: $CONTAINER_URL" + else + echo "Deployment creation failed." + fi + + - name: Catch All Failure Step + if: env.FAILED == '1' + shell: bash + run: | + echo "Workflow failed. See previous steps for details." + exit 1 diff --git a/container maintenance/update-container.sh b/ci-cd automation/update-container.sh similarity index 77% rename from container maintenance/update-container.sh rename to ci-cd automation/update-container.sh index 7fc9342a..484f4996 100644 --- a/container maintenance/update-container.sh +++ b/ci-cd automation/update-container.sh @@ -1,17 +1,26 @@ #!/bin/bash # Script to automatically fetch new contents from a branch, push them to container, and restart intern -# Last Modified on July 17th, 2025 by Maxwell Klema +# Last Modified on August 5th, 2025 by Maxwell Klema # ---------------------------------------- RESET="\033[0m" BOLD="\033[1m" MAGENTA='\033[35m' +outputError() { + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}${MAGENTA}❌ Script Failed. Exiting... ${RESET}" + echo -e "$2" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + exit $1 +} + + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" echo -e "${BOLD}${MAGENTA}🔄 Update Container Contents ${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -if [ "${DEPLOY_ON_START^^}" != "Y" ]; then +if [ -z "${RUNTIME_LANGUAGE^^}" ]; then echo "Skipping container update because there is nothing to update." exit 0 fi @@ -25,57 +34,48 @@ CONTAINER_NAME="${CONTAINER_NAME,,}" if [ -z "$PROJECT_REPOSITORY" ]; then read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY +else + DEPLOY_ON_START="y" fi CheckRepository() { PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY#*github.com/} PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY_SHORTENED%.git} - REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$RROJECT_REPOSITORY) + REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$PROJECT_REPOSITORY_SHORTENED) } - CheckRepository -while [ "$REPOSITORY_EXISTS" != "200" ]; do - echo "⚠️ The repository link you provided, \"$PROJECT_REPOSITORY\" was not valid." - read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY - CheckRepository -done +if [ "$REPOSITORY_EXISTS" != "200" ]; then + outputError 1 "The repository link you provided, \"$PROJECT_REPOSITORY\" was not valid." +fi echo "✅ The repository link you provided, \"$PROJECT_REPOSITORY\", was valid." # Get Project Branch -if [ "$PROJECT_BRANCH" == "" ]; then +if [ -z "$PROJECT_BRANCH" ]; then PROJECT_BRANCH="main" fi REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/$PROJECT_REPOSITORY_SHORTENED/branches/$PROJECT_BRANCH) -REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY/tree/$PROJECT_BRANCH) -while [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; do - echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." - read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH - if [ "PROJECT_BRANCH" == "" ]; then - PROJECT_BRANCH="main" - fi - REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY_SHORTENED/tree/$PROJECT_BRANCH) -done +if [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; then + outputError 1 "The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." +fi # # Get Project Root Directroy -if [ "$PROJECT_ROOT" == "." ] || [ "$PROJECT_ROOT" == "" ]; then +if [ "$PROJECT_ROOT" == "." ] || [ -z "$PROJECT_ROOT" ]; then PROJECT_ROOT="/" fi VALID_PROJECT_ROOT=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateRepo \"$PROJECT_REPOSITORY\" \"$PROJECT_BRANCH\" \"$PROJECT_ROOT\"") -while [ "$VALID_PROJECT_ROOT" == "false" ]; do - echo "⚠️ The root directory you provided, \"$PROJECT_ROOT\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." - read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT - VALID_PROJECT_ROOT=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateRepo \"$PROJECT_REPOSITORY\" \"$PROJECT_BRANCH\" \"$PROJECT_ROOT\"") -done +if [ "$VALID_PROJECT_ROOT" == "false" ]; then + outputError 1 "The root directory you provided, \"$PROJECT_ROOT\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." +fi REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") REPO_BASE_NAME_WITH_OWNER=$(echo "$PROJECT_REPOSITORY" | cut -d'/' -f4) @@ -88,13 +88,34 @@ fi echo "🛎️ Installing Services..." -# SERVICE_COMMANDS=$(ssh -o SendEnv="LINUX_DISTRIBUTION SERVICES CUSTOM_SERVICES REQUIRE_SERVICES" \ -# root@10.15.234.122 \ -# "/root/bin/deployment-scripts/gatherServices.sh true") +if [ -z "$LINUX_DISTRIBUTION" ]; then + LINUX_DISTRIBUTION="debian" +fi + +if [ ! -z "$SERVICES" ] || [ ! -z "$CUSTOM_SERVICES" ]; then + REQUIRE_SERVICES="y" +fi + +SERVICE_COMMANDS=$(ssh -o SendEnv="LINUX_DISTRIBUTION SERVICES CUSTOM_SERVICES REQUIRE_SERVICES" \ + root@10.15.234.122 \ + "/root/bin/deployment-scripts/gatherServices.sh true") + +echo "$SERVICE_COMMANDS" | while read -r line; do + pct exec $CONTAINER_ID -- bash -c "$line | true" > /dev/null 2>&1 +done + +# Change HTTP port if necessary ==== + +if [ ! -z "$HTTP_PORT" ]; then + if [ "$HTTP_PORT" -lt 80 ] || [ "$HTTP_PORT" -gt 60000 ]; then + outputError 1 "Invalid HTTP port: $HTTP_PORT. Must be between 80 and 60000." + fi + ssh root@10.15.20.69 -- \ +"jq \ '.[\"$CONTAINER_NAME\"].ports.http = $HTTP_PORT' \ + /etc/nginx/port_map.json > /tmp/port_map.json.new \ + && mv -f /tmp/port_map.json.new /etc/nginx/port_map.json " +fi -# echo "$SERVICE_COMMANDS" | while read -r line; do -# pct exec $CONTAINER_ID -- bash -c "$line | true" > /dev/null 2>&1 -# done # Clone repository if needed ==== @@ -165,6 +186,10 @@ startComponentPVE2() { } +if [ ! -z "$RUNTIME_LANGUAGE" ] && echo "$RUNTIME_LANGUAGE" | jq . >/dev/null 2>&1; then # If RUNTIME_LANGUAGE is set and is valid JSON + MULTI_COMPONENT="Y" +fi + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') @@ -229,7 +254,6 @@ bash /var/lib/vz/snippets/start_services.sh "$RUNTIME_LANGUAGE_B64" "$GH_ACTION" "$PROJECT_BRANCH" -"$GITHUB_PAT" "$UPDATE_CONTAINER" ) diff --git a/container creation/README.md b/container creation/README.md new file mode 100644 index 00000000..a2ca2a56 --- /dev/null +++ b/container creation/README.md @@ -0,0 +1 @@ +# Container Creation \ No newline at end of file diff --git a/container creation/configureLDAP.sh b/container creation/configureLDAP.sh new file mode 100755 index 00000000..950601e2 --- /dev/null +++ b/container creation/configureLDAP.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Script to connect a container to the LDAP server via SSSD +# Last Modified by Maxwell Klema on July 29th, 2025 +# ----------------------------------------------------- + +# Curl Pown.sh script to install SSSD and configure LDAP +pct enter $CONTAINER_ID < /dev/null 2>&1 && \ +chmod +x pown.sh +EOF + +# Copy .env file to container +ENV_FILE="/var/lib/vz/snippets/.env" +pct enter $CONTAINER_ID < /root/.env +$(cat "$ENV_FILE") +EOT +EOF + +# Run the pown.sh script to configure LDAP +pct exec $CONTAINER_ID -- bash -c "cd /root && ./pown.sh" > /dev/null 2>&1 + +# remove ldap_tls_cert from /etc/sssd/sssd.conf +pct exec $CONTAINER_ID -- sed -i '/ldap_tls_cacert/d' /etc/sssd/sssd.conf > /dev/null 2>&1 + +# Add TLS_REQCERT to never in ROCKY + +if [ "${LINUX_DISTRO^^}" == "ROCKY" ]; then + pct exec $CONTAINER_ID -- bash -c "echo 'TLS_REQCERT never' >> /etc/openldap/ldap.conf" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "authselect select sssd --force" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "systemctl restart sssd" > /dev/null 2>&1 +fi diff --git a/container creation/create-container.sh b/container creation/create-container.sh index dd485893..17d2148b 100644 --- a/container creation/create-container.sh +++ b/container creation/create-container.sh @@ -1,6 +1,7 @@ #!/bin/bash # Script to create the pct container, run register container, and migrate container accordingly. -# Last Modified by July 23rd, 2025 by Maxwell Klema +# Last Modified by August 5th, 2025 by Maxwell Klema +# ----------------------------------------------------- BOLD='\033[1m' BLUE='\033[34m' @@ -33,7 +34,8 @@ echoContainerDetails() { echo -e "📦 ${BLUE}Container ID :${RESET} $CONTAINER_ID" echo -e "🌐 ${MAGENTA}Internal IP :${RESET} $CONTAINER_IP" echo -e "🔗 ${GREEN}Domain Name :${RESET} https://$CONTAINER_NAME.opensource.mieweb.org" - echo -e "🛠️ ${BLUE}SSH Access :${RESET} ssh -p $SSH_PORT root@$CONTAINER_NAME.opensource.mieweb.org" + echo -e "🛠️ ${BLUE}SSH Access :${RESET} ssh -p $SSH_PORT $PROXMOX_USERNAME@$CONTAINER_NAME.opensource.mieweb.org" + echo -e "🔑 ${BLUE}Container Password :${RESET} Your proxmox account password" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" echo -e "${BOLD}${MAGENTA}NOTE: Additional background scripts are being ran in detached terminal sessions.${RESET}" echo -e "${BOLD}${MAGENTA}Wait up to two minutes for all processes to complete.${RESET}" @@ -48,28 +50,27 @@ trap cleanup SIGINT SIGTERM SIGHUP CONTAINER_NAME="${CONTAINER_NAME,,}" CONTAINER_NAME="$1" -CONTAINER_PASSWORD="$2" -GH_ACTION="$3" -HTTP_PORT="$4" -PROXMOX_USERNAME="$5" -PUB_FILE="$6" -PROTOCOL_FILE="$7" +GH_ACTION="$2" +HTTP_PORT="$3" +PROXMOX_USERNAME="$4" +USERNAME_ONLY="${PROXMOX_USERNAME%@*}" +PUB_FILE="$5" +PROTOCOL_FILE="$6" # Deployment ENVS -DEPLOY_ON_START="$8" -PROJECT_REPOSITORY="$9" -PROJECT_BRANCH="${10}" -PROJECT_ROOT="${11}" -INSTALL_COMMAND=$(echo "${12}" | base64 -d) -BUILD_COMMAND=$(echo "${13}" | base64 -d) -START_COMMAND=$(echo "${14}" | base64 -d) -RUNTIME_LANGUAGE=$(echo "${15}" | base64 -d) -ENV_BASE_FOLDER="${16}" -SERVICES_BASE_FILE="${17}" -LINUX_DISTRO="${18}" -MULTI_COMPONENTS="${19}" -ROOT_START_COMMAND="${20}" -GITHUB_PAT="${21}" +DEPLOY_ON_START="$7" +PROJECT_REPOSITORY="$8" +PROJECT_BRANCH="$9" +PROJECT_ROOT="${10}" +INSTALL_COMMAND=$(echo "${11}" | base64 -d) +BUILD_COMMAND=$(echo "${12}" | base64 -d) +START_COMMAND=$(echo "${13}" | base64 -d) +RUNTIME_LANGUAGE=$(echo "${14}" | base64 -d) +ENV_BASE_FOLDER="${15}" +SERVICES_BASE_FILE="${16}" +LINUX_DISTRO="${17}" +MULTI_COMPONENTS="${18}" +ROOT_START_COMMAND="${19}" # Pick the correct template to clone ===== @@ -109,19 +110,16 @@ if [ "${GH_ACTION^^}" != "Y" ]; then pct set $CONTAINER_ID \ --tags "$PROXMOX_USERNAME" \ --tags "$LINUX_DISTRO" \ + --tags "LDAP" \ --onboot 1 > /dev/null 2>&1 pct start $CONTAINER_ID > /dev/null 2>&1 pveum aclmod /vms/$CONTAINER_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser > /dev/null 2>&1 - #pct delete $CONTAINER_ID # Get the Container IP Address and install some packages echo "⏳ Waiting for DHCP to allocate IP address to container..." sleep 5 - - # Set password inside the container - pct exec $CONTAINER_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 else CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$CONTAINER_NAME" '$3 == name {print $1}') fi @@ -133,7 +131,23 @@ if [ -f "/var/lib/vz/snippets/container-public-keys/$PUB_FILE" ]; then rm -rf /var/lib/vz/snippets/container-public-keys/$PUB_FILE > /dev/null 2>&1 fi -CONTAINER_IP=$(pct exec $CONTAINER_ID -- hostname -I | awk '{print $1}') +CONTAINER_IP="" +attempts=0 +max_attempts=10 + +while [[ -z "$CONTAINER_IP" && $attempts -lt $max_attempts ]]; do + CONTAINER_IP=$(pct exec "$CONTAINER_ID" -- hostname -I | awk '{print $1}') + [[ -z "$CONTAINER_IP" ]] && sleep 2 && ((attempts++)) +done + +if [[ -z "$CONTAINER_IP" ]]; then + echo "❌ Timed out waiting for container to get an IP address." + exit 1 +fi + +# Set up SSSD to communicate with LDAP server ==== +echo "⏳ Configuring LDAP connection via SSSD..." +source /var/lib/vz/snippets/helper-scripts/configureLDAP.sh # Attempt to Automatically Deploy Project Inside Container @@ -154,14 +168,15 @@ fi pct exec $CONTAINER_ID -- bash -c "cd /root && touch container-updates.log" # Run Contianer Provision Script to add container to port_map.json - +echo "⏳ Running Container Provision Script..." if [ -f "/var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE" ]; then - /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE - rm -rf /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE > /dev/null 2>&1 + /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE "$USERNAME_ONLY" + rm -rf /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE > /dev/null 2>&1 else - /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT + /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT "" "$PROXMOX_USERNAME" fi + SSH_PORT=$(iptables -t nat -S PREROUTING | grep "to-destination $CONTAINER_IP:22" | awk -F'--dport ' '{print $2}' | awk '{print $1}' | head -n 1 || true) # Output container details and start services if necessary ===== @@ -189,11 +204,10 @@ bash /var/lib/vz/snippets/start_services.sh "$RUNTIME_LANGUAGE_B64" "$GH_ACTION" "$PROJECT_BRANCH" -"$GITHUB_PAT" ) # Safely quote each argument for the shell QUOTED_CMD=$(printf ' %q' "${CMD[@]}") tmux new-session -d -s "$CONTAINER_NAME" "$QUOTED_CMD" -exit 0 \ No newline at end of file +exit 0 diff --git a/container creation/deployOnStart.sh b/container creation/deployOnStart.sh index ae1bb486..82d01a62 100755 --- a/container creation/deployOnStart.sh +++ b/container creation/deployOnStart.sh @@ -13,7 +13,9 @@ echo "Repo base name: $REPO_BASE_NAME" pct enter $CONTAINER_ID < /dev/null +git clone $PROJECT_REPOSITORY && \ +cd /root/$REPO_BASE_NAME && \ +git checkout $PROJECT_BRANCH > /dev/null else cd /root/$REPO_BASE_NAME && git fetch && git pull && \ git checkout $PROJECT_BRANCH @@ -26,19 +28,21 @@ pct exec $CONTAINER_ID -- bash -c "chmod 700 ~/.bashrc" # enable full R/W/X perm ENV_BASE_FOLDER="/var/lib/vz/snippets/container-env-vars/${ENV_BASE_FOLDER}" -if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then - for FILE in $ENV_BASE_FOLDER/*; do - FILE_BASENAME=$(basename "$FILE") - FILE_NAME="${FILE_BASENAME%.*}" - ENV_ROUTE=$(echo "$FILE_NAME" | tr '_' '/') # acts as the route to the correct folder to place .env file in. - - ENV_VARS=$(cat $ENV_BASE_FOLDER/$FILE_BASENAME) - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$ENV_ROUTE && echo "$ENV_VARS" > .env" > /dev/null 2>&1 - done -else - ENV_FOLDER_BASE_NAME=$(basename "$ENV_BASE_FOLDER") - ENV_VARS=$(cat $ENV_BASE_FOLDER/$ENV_FOLDER_BASE_NAME.txt) - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && echo "$ENV_VARS" > .env" > /dev/null 2>&1 +if [ ! -d "$ENV_BASE_FOLDER"]; then + if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then + for FILE in $ENV_BASE_FOLDER/*; do + FILE_BASENAME=$(basename "$FILE") + FILE_NAME="${FILE_BASENAME%.*}" + ENV_ROUTE=$(echo "$FILE_NAME" | tr '_' '/') # acts as the route to the correct folder to place .env file in. + + ENV_VARS=$(cat $ENV_BASE_FOLDER/$FILE_BASENAME) + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$ENV_ROUTE && echo "$ENV_VARS" > .env" > /dev/null 2>&1 + done + else + ENV_FOLDER_BASE_NAME=$(basename "$ENV_BASE_FOLDER") + ENV_VARS=$(cat $ENV_BASE_FOLDER/$ENV_FOLDER_BASE_NAME.txt || true) + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && echo "$ENV_VARS" > .env" > /dev/null 2>&1 + fi fi # Run Installation Commands ==== diff --git a/deployment-scripts/gatherEnvVars.sh b/container creation/deployment-scripts/gatherEnvVars.sh similarity index 74% rename from deployment-scripts/gatherEnvVars.sh rename to container creation/deployment-scripts/gatherEnvVars.sh index cd9b9d5a..9453c065 100644 --- a/deployment-scripts/gatherEnvVars.sh +++ b/container creation/deployment-scripts/gatherEnvVars.sh @@ -8,7 +8,13 @@ gatherEnvVars(){ read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE while [ "$ENV_VAR_KEY" == "" ] || [ "$ENV_VAR_VALUE" == "" ]; do + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Key and value cannot be empty. Please try again." + writeLog "Empty environment variable key or value entered (GH_ACTION mode)" + exit 15 + fi echo "⚠️ Key or value cannot be empty. Try again." + writeLog "Empty environment variable key or value entered" read -p "🔑 Enter Environment Variable Key → " ENV_VAR_KEY read -p "🔑 Enter Environment Variable Value → " ENV_VAR_VALUE done @@ -28,9 +34,16 @@ fi while [ "${REQUIRE_ENV_VARS^^}" != "Y" ] && [ "${REQUIRE_ENV_VARS^^}" != "N" ] && [ "${REQUIRE_ENV_VARS^^}" != "" ]; do echo "⚠️ Invalid option. Please try again." + writeLog "Invalid environment variables requirement option entered: $REQUIRE_ENV_VARS" read -p "🔑 Does your application require environment variables? (y/n) → " REQUIRE_ENV_VARS done +if [ "${GH_ACTION^^}" == "Y" ]; then + if [ ! -z "$CONTAINER_ENV_VARS" ]; then + REQUIRE_ENV_VARS="Y" + fi +fi + if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then # generate random temp .env folder to store all env files for different components RANDOM_NUM=$(shuf -i 100000-999999 -n 1) @@ -51,8 +64,14 @@ if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then addComponent "$key" done else + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." + writeLog "Invalid JSON in CONTAINER_ENV_VARS (GH_ACTION mode)" + exit 16 + fi echo "⚠️ Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." - exit 10 + writeLog "Invalid JSON in CONTAINER_ENV_VARS" + exit 16 fi else # No Environment Variables gatherComponentDir "Enter the path of your component to enter environment variables" @@ -71,12 +90,19 @@ if [ "${REQUIRE_ENV_VARS^^}" == "Y" ]; then ENV_FILE="env_$RANDOM_NUM.txt" ENV_FILE_PATH="/root/bin/env/$ENV_FOLDER/$ENV_FILE" touch "$ENV_FILE_PATH" + if [ ! -z "$CONTAINER_ENV_VARS" ]; then # Environment Variables if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success echo "$CONTAINER_ENV_VARS " | jq -r 'to_entries[] | "\(.key)=\(.value)"' > "$ENV_FILE_PATH" #k=v pairs else + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." + writeLog "Invalid JSON in CONTAINER_ENV_VARS for single component (GH_ACTION mode)" + exit 16 + fi echo "⚠️ Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." - exit 10 + writeLog "Invalid JSON in CONTAINER_ENV_VARS for single component" + exit 16 fi else # No Environment Variables gatherEnvVars "$ENV_FILE_PATH" diff --git a/deployment-scripts/gatherRuntimeLangs.sh b/container creation/deployment-scripts/gatherRuntimeLangs.sh old mode 100644 new mode 100755 similarity index 63% rename from deployment-scripts/gatherRuntimeLangs.sh rename to container creation/deployment-scripts/gatherRuntimeLangs.sh index 1331610f..508eff5c --- a/deployment-scripts/gatherRuntimeLangs.sh +++ b/container creation/deployment-scripts/gatherRuntimeLangs.sh @@ -9,6 +9,12 @@ gatherRunTime() { while [ "${RUNTIME_LANGUAGE^^}" != "NODEJS" ] && [ "${RUNTIME_LANGUAGE^^}" != "PYTHON" ]; do echo "⚠️ Sorry, that runtime environment is not yet supported. Only \"nodejs\" and \"python\" are currently supported." + writeLog "Unsupported runtime environment entered: $RUNTIME_LANGUAGE for component: $COMPONENT_PATH" + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "⚠️ Sorry, that runtime environment is not yet supported. Only \"nodejs\" and \"python\" are currently supported." + writeLog "Unsupported runtime environment entered: $RUNTIME_LANGUAGE (GH_ACTION mode)" + exit 17 + fi read -p "🖥️ Enter the underlying runtime environment for \"$COMPONENT_PATH\" (e.g., 'nodejs', 'python') → " RUNTIME_LANGUAGE done } @@ -28,6 +34,7 @@ removeFromList() { UNIQUE_COMPONENTS_CLONE=("${UNIQUE_COMPONENTS[@]}") RUNTIME_LANGUAGE_DICT={} + if [ "${MULTI_COMPONENT^^}" == 'Y' ]; then if [ ! -z "$RUNTIME_LANGUAGE" ]; then # Environment Variable Passed if echo "$RUNTIME_LANGUAGE" | jq -e > /dev/null 2>&1; then # Valid JSON @@ -35,12 +42,24 @@ if [ "${MULTI_COMPONENT^^}" == 'Y' ]; then removeFromList "$key" done if [ ${#UNIQUE_COMPONENTS_CLONE[@]} -gt 0 ]; then #if there are still components in the list, then not all runtimes were provided, so exit on error + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "You did not provide runtime languages for these components: \"${UNIQUE_COMPONENTS_CLONE[@]}\"." + writeLog "Missing runtime languages for components: ${UNIQUE_COMPONENTS_CLONE[@]} (GH_ACTION mode)" + exit 18 + fi echo "⚠️ You did not provide runtime languages for these components: \"${UNIQUE_COMPONENTS_CLONE[@]}\"." - exit 11 + writeLog "Missing runtime languages for components: ${UNIQUE_COMPONENTS_CLONE[@]}" + exit 18 fi else + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Your \"$RUNTIME_LANGUAGE\" is not valid JSON. Please re-format and try again." + writeLog "Invalid JSON in RUNTIME_LANGUAGE (GH_ACTION mode)" + exit 16 + fi echo "⚠️ Your \"$RUNTIME_LANGUAGE\" is not valid JSON. Please re-format and try again." - exit 10 + writeLog "Invalid JSON in RUNTIME_LANGUAGE" + exit 16 fi else # No Environment Variable Passed for CURRENT in "${UNIQUE_COMPONENTS[@]}"; do @@ -51,7 +70,7 @@ if [ "${MULTI_COMPONENT^^}" == 'Y' ]; then fi else if [ ! -z "$RUNTIME_LANGUAGE" ]; then - RUNTIME_LANGUAGE="true" + RT_ENV_VAR="true" fi gatherRunTime "$PROJECT_REPOSITORY" fi \ No newline at end of file diff --git a/container creation/deployment-scripts/gatherServices.sh b/container creation/deployment-scripts/gatherServices.sh new file mode 100755 index 00000000..1cc06c39 --- /dev/null +++ b/container creation/deployment-scripts/gatherServices.sh @@ -0,0 +1,158 @@ +SERVICE_MAP="/root/bin/services/service_map_$LINUX_DISTRIBUTION.json" +APPENDED_SERVICES=() + +# Helper function to check if a user has added the same service twice +serviceExists() { + SERVICE="$1" + for CURRENT in "${APPENDED_SERVICES[@]}"; do + if [ "${SERVICE,,}" == "${CURRENT,,}" ]; then + return 0 + fi + done + return 1 +} + +processService() { + local SERVICE="$1" + local MODE="$2" # "batch" or "single" + + SERVICE_IN_MAP=$(jq -r --arg key "${SERVICE,,}" '.[$key] // empty' "$SERVICE_MAP") + if serviceExists "$SERVICE"; then + if [ "$MODE" = "batch" ]; then + return 0 # skip to next in batch mode + else + echo "⚠️ You already added \"$SERVICE\" as a service. Please try again." + writeLog "Duplicate service attempted: $SERVICE" + return 0 + fi + elif [ "${SERVICE^^}" != "C" ] && [ "${SERVICE^^}" != "" ] && [ -n "$SERVICE_IN_MAP" ]; then + jq -r --arg key "${SERVICE,,}" '.[$key][]' "$SERVICE_MAP" >> "$TEMP_SERVICES_FILE_PATH" + echo "sudo systemctl daemon-reload" >> "$TEMP_SERVICES_FILE_PATH" + echo "✅ ${SERVICE^^} added to your container." + APPENDED_SERVICES+=("${SERVICE^^}") + elif [ "${SERVICE^^}" == "C" ]; then + appendCustomService + elif [ "${SERVICE^^}" != "" ]; then + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "⚠️ Service \"$SERVICE\" does not exist." + writeLog "Invalid service entered: $SERVICE (GH_ACTION mode)" + exit 20 + fi + echo "⚠️ Service \"$SERVICE\" does not exist." + writeLog "Invalid service entered: $SERVICE" + [ "$MODE" = "batch" ] && exit 20 + fi +} + +# Helper function to append a new service to a container +appendService() { + if [ ! -z "$SERVICES" ]; then + for SERVICE in $(echo "$SERVICES" | jq -r '.[]'); do + processService "$SERVICE" "batch" + done + else + read -p "➡️ Enter the name of a service to add to your container or type \"C\" to set up a custom service installation (Enter to exit) → " SERVICE + processService "$SERVICE" "single" + fi +} + +appendCustomService() { + # If there is an env variable for custom services, iterate through each command and append it to temporary services file + if [ ! -z "$CUSTOM_SERVICES" ]; then + echo "$CUSTOM_SERVICES" | jq -c -r '.[]' | while read -r CUSTOM_SERVICE; do + echo "$CUSTOM_SERVICE" | jq -c -r '.[]' | while read -r CUSTOM_SERVICE_COMMAND; do + if [ ! -z "$CUSTOM_SERVICE_COMMAND" ]; then + echo "$CUSTOM_SERVICE_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" + else + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "⚠️ Custom Service Installation Command cannot be empty in \"$CUSTOM_SERVICE\"." + writeLog "Empty custom service command in: $CUSTOM_SERVICE (GH_ACTION mode)" + exit 21 + fi + echo "⚠️ Command cannot be empty." + writeLog "Empty custom service command in: $CUSTOM_SERVICE" + exit 21; + fi + done + done + echo "✅ Custom Services appended." + else + echo "🛎️ Configuring Custom Service Installation. For each prompt, enter a command that is a part of the installation process for your service on Debian Bookworm. Do not forget to enable and start the service at the end. Once you have entered all of your commands, press enter to continue" + COMMAND_NUM=1 + read -p "➡️ Enter Command $COMMAND_NUM: " CUSTOM_COMMAND + + echo "$CUSTOM_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" + + while [ "${CUSTOM_COMMAND^^}" != "" ]; do + ((COMMAND_NUM++)) + read -p "➡️ Enter Command $COMMAND_NUM: " CUSTOM_COMMAND + echo "$CUSTOM_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" + done + fi +} + +# Helper function to see if a user wants to set up a custom service +setUpService() { + read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION + while [ "${SETUP_CUSTOM_SERVICE_INSTALLATION^^}" != "Y" ] && [ "${SETUP_CUSTOM_SERVICE_INSTALLATION^^}" != "N" ] && [ "${SETUP_CUSTOM_SERVICE_INSTALLATION^^}" != "" ]; do + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "⚠️ Invalid custom service installation option. Please try again." + writeLog "Invalid custom service installation option entered: $SETUP_CUSTOM_SERVICE_INSTALLATION (GH_ACTION mode)" + exit 22 + fi + echo "⚠️ Invalid option. Please try again." + writeLog "Invalid custom service installation option entered: $SETUP_CUSTOM_SERVICE_INSTALLATION" + read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION + done +} + +if [ -z "$REQUIRE_SERVICES" ]; then + read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES +fi + +while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do + echo "⚠️ Invalid option. Please try again." + writeLog "Invalid service requirement option entered: $REQUIRE_SERVICES" + read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES +done + +if [ "${GH_ACTION^^}" == "Y" ]; then + if [ ! -z "$SERVICES" ] || [ ! -z "$CUSTOM_SERVICES" ]; then + REQUIRE_SERVICES="Y" + fi +fi + +if [ "${REQUIRE_SERVICES^^}" == "Y" ]; then + + # Generate random (temporary) file to store install commands for needed services + RANDOM_NUM=$(shuf -i 100000-999999 -n 1) + SERVICES_FILE="services_$RANDOM_NUM.txt" + TEMP_SERVICES_FILE_PATH="/root/bin/services/$SERVICES_FILE" + touch "$TEMP_SERVICES_FILE_PATH" + + appendService + while [ "${SERVICE^^}" != "" ] || [ ! -z "$SERVICES" ]; do + if [ -z "$SERVICES" ]; then + appendService + else + if [ ! -z "$CUSTOM_SERVICES" ]; then # assumes both services and custom services passed as ENV vars + appendCustomService + else # custom services not passed as ENV var, so must prompt the user for their custom services + setUpService + while [ "${SETUP_CUSTOM_SERVICE_INSTALLATION^^}" == "Y" ]; do + appendCustomService + setUpService + done + fi + break + fi + done +fi + +# Used for updating container services in GH Actions + +UPDATING_CONTAINER="$1" +if [ "$UPDATING_CONTAINER" == "true" ]; then + cat "$TEMP_SERVICES_FILE_PATH" + rm -rf "$TEMP_SERVICES_FILE_PATH" +fi \ No newline at end of file diff --git a/deployment-scripts/gatherSetupCommands.sh b/container creation/deployment-scripts/gatherSetupCommands.sh similarity index 83% rename from deployment-scripts/gatherSetupCommands.sh rename to container creation/deployment-scripts/gatherSetupCommands.sh index 4aaf6478..6ff40ecb 100644 --- a/deployment-scripts/gatherSetupCommands.sh +++ b/container creation/deployment-scripts/gatherSetupCommands.sh @@ -1,6 +1,6 @@ #!/bin/bash # This function gathers start up commands, such as build, install, and start, for both single and multiple component applications -# Last Modified by Maxwell Klema on July 16th, 2025 +# Last Modified by Maxwell Klema on July 15th, 2025 # --------------------------------------------- gatherSetupCommands() { @@ -19,7 +19,13 @@ gatherSetupCommands() { addComponent "$key" done else + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Your \"$TYPE_COMMAND\" is not valid JSON. Please re-format and try again." + writeLog "Invalid JSON in $TYPE_COMMAND (GH_ACTION mode)" + exit 10 + fi echo "⚠️ Your \"$TYPE_COMMAND\" is not valid JSON. Please re-format and try again." + writeLog "Invalid JSON in $TYPE_COMMAND" exit 10 fi else # No Environment Variable Passed diff --git a/container creation/get-deployment-details.sh b/container creation/get-deployment-details.sh index 5d1e9631..2f5f4215 100755 --- a/container creation/get-deployment-details.sh +++ b/container creation/get-deployment-details.sh @@ -1,6 +1,6 @@ #!/bin/bash # Helper script to gather project details for automatic deployment -# Modified July 17th, 2025 by Maxwell Klema +# Modified August 5th, 2025 by Maxwell Klema # ------------------------------------------ # Define color variables (works on both light and dark backgrounds) @@ -12,60 +12,94 @@ echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━ echo -e "${BOLD}${MAGENTA}🌐 Let's Get Your Project Automatically Deployed ${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +writeLog "Starting deploy application script" + # Get and validate project repository ======== if [ -z "$PROJECT_REPOSITORY" ]; then read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY + writeLog "Prompted for project repository" fi CheckRepository() { PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY#*github.com/} PROJECT_REPOSITORY_SHORTENED=${PROJECT_REPOSITORY_SHORTENED%.git} - REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$RROJECT_REPOSITORY) + REPOSITORY_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$PROJECT_REPOSITORY_SHORTENED) + writeLog "Checking repository existence for $PROJECT_REPOSITORY_SHORTENED" } CheckRepository while [ "$REPOSITORY_EXISTS" != "200" ]; do + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Invalid Repository Link. Make sure your repository is private." + writeLog "Invalid repository link entered: $PROJECT_REPOSITORY (GH_ACTION mode)" + exit 10 + fi echo "⚠️ The repository link you provided, \"$PROJECT_REPOSITORY\" was not valid." + writeLog "Invalid repository link entered: $PROJECT_REPOSITORY" read -p "🚀 Paste the link to your project repository → " PROJECT_REPOSITORY CheckRepository done +writeLog "Repository validated: $PROJECT_REPOSITORY" + # Get Repository Branch ======== if [ -z "$PROJECT_BRANCH" ]; then read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH + writeLog "Prompted for project branch" fi if [ "$PROJECT_BRANCH" == "" ]; then PROJECT_BRANCH="main" + writeLog "Using default branch: main" fi -REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY/tree/$PROJECT_BRANCH) +REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$PROJECT_REPOSITORY_SHORTENED/tree/$PROJECT_BRANCH) +writeLog "Checking branch existence for $PROJECT_BRANCH" + while [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; do + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Invalid Branch. Make sure your branch exists on the repository." + writeLog "Invalid branch entered: $PROJECT_BRANCH (GH_ACTION mode)" + exit 11 + fi echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." + writeLog "Invalid branch entered: $PROJECT_BRANCH" read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH - if [ "PROJECT_BRANCH" == "" ]; then - PROJECT_BRANCH="main" + if [ "$PROJECT_BRANCH" == "" ]; then + PROJECT_BRANCH="main" fi - REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" $PROJECT_REPOSITORY_SHORTENED/tree/$PROJECT_BRANCH) + REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$PROJECT_REPOSITORY_SHORTENED/tree/$PROJECT_BRANCH) done +writeLog "Branch validated: $PROJECT_BRANCH" + # Get Project Root Directory ======== if [ -z "$PROJECT_ROOT" ]; then read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT + writeLog "Prompted for project root directory" fi VALID_PROJECT_ROOT=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") +writeLog "Validating project root directory: $PROJECT_ROOT" while [ "$VALID_PROJECT_ROOT" == "false" ]; do + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Invalid Project Root Directory. Make sure your directory exists on the repository." + writeLog "Invalid project root directory entered: $PROJECT_ROOT (GH_ACTION mode)" + exit 12 + fi echo "⚠️ The root directory you provided, \"$PROJECT_ROOT\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." + writeLog "Invalid project root directory entered: $PROJECT_ROOT" read -p "📁 Enter the project root directory (relative to repository root directory, or leave blank for root directory) → " PROJECT_ROOT VALID_PROJECT_ROOT=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$PROJECT_ROOT") done +writeLog "Project root directory validated: $PROJECT_ROOT" + # Remove forward slash if [[ "$PROJECT_ROOT" == "/*" ]]; then PROJECT_ROOT="${PROJECT_ROOT:1}" @@ -75,13 +109,28 @@ fi if [ -z "$MULTI_COMPONENT" ]; then read -p "🔗 Does your app consist of multiple components that run independently, i.e. seperate frontend and backend (y/n) → " MULTI_COMPONENT + writeLog "Prompted for multi-component option" fi while [ "${MULTI_COMPONENT^^}" != "Y" ] && [ "${MULTI_COMPONENT^^}" != "N" ] && [ "${MULTI_COMPONENT^^}" != "" ]; do + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Invalid option for MULTI_COMPONENT. It must be 'y' or 'n'. Please try again." + writeLog "Invalid multi-component option entered: $MULTI_COMPONENT (GH_ACTION mode)" + exit 13 + fi echo "⚠️ Invalid option. Please try again." + writeLog "Invalid multi-component option entered: $MULTI_COMPONENT" read -p "🔗 Does your app consist of multiple components that run independently, i.e. seperate frontend and backend (y/n) → " MULTI_COMPONENT done +if [ "${GH_ACTION^^}" == "Y" ]; then + if [ ! -z "$RUNTIME_LANGUAGE" ] && echo "$RUNTIME_LANGUAGE" | jq . >/dev/null 2>&1; then # If RUNTIME_LANGUAGE is set and is valid JSON + MULTI_COMPONENT="Y" + fi +fi + +writeLog "Multi-component option set to: $MULTI_COMPONENT" + # Gather Deployment Commands ======== # Helper functions to gather and validate component directory @@ -90,22 +139,35 @@ gatherComponentDir() { COMPONENT_PATH="$2" if [ -z "$COMPONENT_PATH" ]; then read -p "$1, relative to project root directory (To Continue, Press Enter) → " COMPONENT_PATH + writeLog "Prompted for component directory: $1" fi # Check that component path is valid VALID_COMPONENT_PATH=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$COMPONENT_PATH") + writeLog "Validating component path: $COMPONENT_PATH" + while [ "$VALID_COMPONENT_PATH" == "false" ] && [ "$COMPONENT_PATH" != "" ]; do + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Invalid Component Path: \"$COMPONENT_PATH\". Make sure your path exists on the repository." + writeLog "Invalid component path entered: $COMPONENT_PATH (GH_ACTION mode)" + exit 14 + fi echo "⚠️ The component path you entered, \"$COMPONENT_PATH\", does not exist on branch, \"$PROJECT_BRANCH\", on repository at \"$PROJECT_REPOSITORY\"." + writeLog "Invalid component path entered: $COMPONENT_PATH" if [ -z "$2" ]; then read -p "$1, relative to project root directory (To Continue, Press Enter) → " COMPONENT_PATH VALID_COMPONENT_PATH=$(node /root/bin/js/runner.js authenticateRepo "$PROJECT_REPOSITORY" "$PROJECT_BRANCH" "$COMPONENT_PATH") else - exit 9 + exit 14 fi done if [[ "$COMPONENT_PATH" == /* ]]; then COMPONENT_PATH="${COMPONENT_PATH:1}" # remove leading slash fi + + if [ "$COMPONENT_PATH" != "" ]; then + writeLog "Component path validated: $COMPONENT_PATH" + fi } UNIQUE_COMPONENTS=() @@ -119,151 +181,42 @@ addComponent() { fi done UNIQUE_COMPONENTS+=("$COMPONENT") + writeLog "Added component: $COMPONENT" } +writeLog "Sourcing setup commands script" source /root/bin/deployment-scripts/gatherSetupCommands.sh # Function to gather build, install, and start commands +writeLog "Sourcing environment variables script" source /root/bin/deployment-scripts/gatherEnvVars.sh # Gather Environment Variables + +writeLog "Gathering build commands" gatherSetupCommands "BUILD" "🏗️ Enter the build command (leave blank if no build command) → " # Gather Build Command(s) -gatherSetupCommands "INSTALL" "📦 Enter the install command (e.g., 'npm install') → " # Gather Install Command(s)echo "$INSTALL_COMMAND" -gatherSetupCommands "START" "🚦 Enter the start command (e.g., 'npm start', 'python app.py') → " # Gather Start Command(s) +writeLog "Gathering install commands" +gatherSetupCommands "INSTALL" "📦 Enter the install command (e.g., 'npm install') → " # Gather Install Command(s) + +writeLog "Gathering start commands" +gatherSetupCommands "START" "🚦 Enter the start command (e.g., 'npm start', 'python app.py') → " # Gather Start Command(s) if [ "${MULTI_COMPONENT^^}" == "Y" ]; then if [ -z "$ROOT_START_COMMAND" ]; then read -p "📍 If your container requires a start command at the root directory, i.e. Docker run, enter it here (leave blank for no command) → " ROOT_START_COMMAND + writeLog "Prompted for root start command" + fi + if [ "$ROOT_START_COMMAND" != "" ]; then + writeLog "Root start command set: $ROOT_START_COMMAND" fi fi # Get Runtime Language ======== +writeLog "Sourcing runtime languages script" source /root/bin/deployment-scripts/gatherRuntimeLangs.sh # Get Services ======== +writeLog "Sourcing services script" +source /root/bin/deployment-scripts/gatherServices.sh -SERVICE_MAP="/root/bin/services/service_map_$LINUX_DISTRIBUTION.json" -APPENDED_SERVICES=() - -# Helper function to check if a user has added the same service twice -serviceExists() { - SERVICE="$1" - for CURRENT in "${APPENDED_SERVICES[@]}"; do - if [ "${SERVICE,,}" == "${CURRENT,,}" ]; then - return 0 - fi - done - return 1 -} - -processService() { - local SERVICE="$1" - local MODE="$2" # "batch" or "single" - - SERVICE_IN_MAP=$(jq -r --arg key "${SERVICE,,}" '.[$key] // empty' "$SERVICE_MAP") - if serviceExists "$SERVICE"; then - if [ "$MODE" = "batch" ]; then - return 0 # skip to next in batch mode - else - echo "⚠️ You already added \"$SERVICE\" as a service. Please try again." - return 0 - fi - elif [ "${SERVICE^^}" != "C" ] && [ "${SERVICE^^}" != "" ] && [ -n "$SERVICE_IN_MAP" ]; then - jq -r --arg key "${SERVICE,,}" '.[$key][]' "$SERVICE_MAP" >> "$TEMP_SERVICES_FILE_PATH" - echo "sudo systemctl daemon-reload" >> "$TEMP_SERVICES_FILE_PATH" - echo "✅ ${SERVICE^^} added to your container." - APPENDED_SERVICES+=("${SERVICE^^}") - elif [ "${SERVICE^^}" == "C" ]; then - appendCustomService - elif [ "${SERVICE^^}" != "" ]; then - echo "⚠️ Service \"$SERVICE\" does not exist." - [ "$MODE" = "batch" ] && exit 20 - fi -} - -# Helper function to append a new service to a container -appendService() { - if [ ! -z "$SERVICES" ]; then - for SERVICE in $(echo "$SERVICES" | jq -r '.[]'); do - processService "$SERVICE" "batch" - done - else - read -p "➡️ Enter the name of a service to add to your container or type \"C\" to set up a custom service installation (Enter to exit) → " SERVICE - processService "$SERVICE" "single" - fi -} - -appendCustomService() { - # If there is an env variable for custom services, iterate through each command and append it to temporary services file - if [ ! -z "$CUSTOM_SERVICES" ]; then - echo "$CUSTOM_SERVICES" | jq -c -r '.[]' | while read -r CUSTOM_SERVICE; do - echo "$CUSTOM_SERVICE" | jq -c -r '.[]' | while read -r CUSTOM_SERVICE_COMMAND; do - if [ ! -z "$CUSTOM_SERVICE_COMMAND" ]; then - echo "$CUSTOM_SERVICE_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" - else - echo "⚠️ Command cannot be empty." - exit 21; - fi - done - done - echo "✅ Custom Services appended." - else - echo "🛎️ Configuring Custom Service Installation. For each prompt, enter a command that is a part of the installation process for your service on Debian Bookworm. Do not forget to enable and start the service at the end. Once you have entered all of your commands, press enter to continue" - COMMAND_NUM=1 - read -p "➡️ Enter Command $COMMAND_NUM: " CUSTOM_COMMAND - - echo "$CUSTOM_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" - - while [ "${CUSTOM_COMMAND^^}" != "" ]; do - ((COMMAND_NUM++)) - read -p "➡️ Enter Command $COMMAND_NUM: " CUSTOM_COMMAND - echo "$CUSTOM_COMMAND" >> "$TEMP_SERVICES_FILE_PATH" - done - fi -} - -# Helper function to see if a user wants to set up a custom service -setUpService() { - read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION - while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do - echo "⚠️ Invalid option. Please try again." - read -p "🛎️ Do you wish to set up a custom service installation? (y/n) " SETUP_CUSTOM_SERVICE_INSTALLATION - done -} - -if [ -z "$REQUIRE_SERVICES" ]; then - read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES -fi - -while [ "${REQUIRE_SERVICES^^}" != "Y" ] && [ "${REQUIRE_SERVICES^^}" != "N" ] && [ "${REQUIRE_SERVICES^^}" != "" ]; do - echo "⚠️ Invalid option. Please try again." - read -p "🛎️ Does your application require special services (i.e. Docker, MongoDB, etc.) to run on the container? (y/n) → " REQUIRE_SERVICES -done - -if [ "${REQUIRE_SERVICES^^}" == "Y" ]; then - - # Generate random (temporary) file to store install commands for needed services - RANDOM_NUM=$(shuf -i 100000-999999 -n 1) - SERVICES_FILE="services_$RANDOM_NUM.txt" - TEMP_SERVICES_FILE_PATH="/root/bin/services/$SERVICES_FILE" - touch "$TEMP_SERVICES_FILE_PATH" - - appendService - while [ "${SERVICE^^}" != "" ] || [ ! -z "$SERVICES" ]; do - if [ -z "$SERVICES" ]; then - appendService - else - if [ ! -z "$CUSTOM_SERVICES" ]; then # assumes both services and custom services passed as ENV vars - appendCustomService - else # custom services not passed as ENV var, so must prompt the user for their custom services - setUpService - while [ "${SETUP_CUSTOM_SERVICE_INSTALLATION^^}" == "Y" ]; do - appendCustomService - setUpService - done - fi - break - fi - done -fi - -echo -e "\n✅ Deployment Process Finished.\n" +writeLog "Deployment process finished successfully" +echo -e "\n✅ Deployment Process Finished.\n" \ No newline at end of file diff --git a/container creation/get-lxc-container-details.sh b/container creation/get-lxc-container-details.sh index 3c746421..148900f2 100644 --- a/container creation/get-lxc-container-details.sh +++ b/container creation/get-lxc-container-details.sh @@ -1,8 +1,14 @@ #!/bin/bash # Main Container Creation Script -# Modified July 17th, 2025 by Maxwell Klema +# Modified July 28th, 2025 by Maxwell Klema # ------------------------------------------ +LOG_FILE="/var/log/create-container.log" + +writeLog() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')]: $1" >> "$LOG_FILE" +} + # Define color variables (works on both light and dark backgrounds) RESET="\033[0m" BOLD="\033[1m" @@ -14,114 +20,79 @@ echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━ # Authenticate User (Only Valid Users can Create Containers) +outputError() { + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}${MAGENTA}❌ Script Failed. Exiting... ${RESET}" + echo -e "$1" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +} + +writeLog "Starting Container Creation Script" + if [ -z "$PROXMOX_USERNAME" ]; then - read -p "Enter Proxmox Username → " PROXMOX_USERNAME + read -p "Enter Proxmox Username → " PROXMOX_USERNAME fi if [ -z "$PROXMOX_PASSWORD" ]; then - read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD - echo "" + read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD + echo "" fi USER_AUTHENTICATED=$(node /root/bin/js/runner.js authenticateUser "$PROXMOX_USERNAME" "$PROXMOX_PASSWORD") -RETRIES=3 while [ $USER_AUTHENTICATED == 'false' ]; do - if [ $RETRIES -gt 0 ]; then - echo "❌ Authentication Failed. Try Again" - read -p "Enter Proxmox Username → " PROXMOX_USERNAME - read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD - echo "" - - USER_AUTHENTICATED=$(node /root/bin/js/runner.js authenticateUser "$PROXMOX_USERNAME" "$PROXMOX_PASSWORD") - RETRIES=$(($RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 2 - fi + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Invalid Proxmox Credentials." + writeLog "Invalid Proxmox credentials entered for user: $PROXMOX_USERNAME (GH_ACTION mode)" + exit 2 + fi + echo "❌ Authentication Failed. Try Again" + writeLog "Invalid Proxmox credentials entered for user: $PROXMOX_USERNAME" + read -p "Enter Proxmox Username → " PROXMOX_USERNAME + read -sp "Enter Proxmox Password → " PROXMOX_PASSWORD + echo "" + + USER_AUTHENTICATED=$(node /root/bin/js/runner.js authenticateUser "$PROXMOX_USERNAME" "$PROXMOX_PASSWORD") done echo "🎉 Your proxmox account, $PROXMOX_USERNAME@pve, has been authenticated" -# Gather Container Hostname (hostname.opensource.mieweb.org) +# Gather Container Hostname (hostname.opensource.mieweb.org) ===== if [ -z "$CONTAINER_NAME" ]; then - read -p "Enter Application Name (One-Word) → " CONTAINER_NAME + read -p "Enter Application Name (One-Word) → " CONTAINER_NAME fi CONTAINER_NAME="${CONTAINER_NAME,,}" #convert to lowercase HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") -HOST_NAME_RETRIES=10 while [[ $HOST_NAME_EXISTS == 'true' ]] || ! [[ "$CONTAINER_NAME" =~ ^[A-Za-z0-9-]+$ ]]; do - if [ $HOST_NAME_RETRIES -gt 0 ]; then - echo "Sorry! Either that name has already been registered or your hostname is ill-formatted. Try another name" - read -p "Enter Application Name (One-Word) → " CONTAINER_NAME - HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") - HOST_NAME_RETRIES=$(($HOST_NAME_RETRIES-1)) - CONTAINER_NAME="${CONTAINER_NAME,,}" - else - echo "Too many incorrect attempts. Exiting..." - exit 3 - fi + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Invalid Container Hostname." + writeLog "Invalid container hostname entered: $CONTAINER_NAME (GH_ACTION mode)" + exit 3 + fi + echo "Sorry! Either that name has already been registered or your hostname is ill-formatted. Try another name" + writeLog "Invalid container hostname entered: $CONTAINER_NAME (already exists or ill-formatted)" + read -p "Enter Application Name (One-Word) → " CONTAINER_NAME + HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") + CONTAINER_NAME="${CONTAINER_NAME,,}" done echo "✅ $CONTAINER_NAME is available" -# Gather Container Password -PASSWORD_RETRIES=10 - -if [ -z "$CONTAINER_PASSWORD" ]; then - read -sp "Enter Container Password → " CONTAINER_PASSWORD - echo - read -sp "Confirm Container Password → " CONFIRM_PASSWORD - echo - - while [[ "$CONFIRM_PASSWORD" != "$CONTAINER_PASSWORD" || ${#CONTAINER_PASSWORD} -lt 8 ]]; do - if [ $PASSWORD_RETRIES -gt 0 ]; then - echo "Sorry, try again. Ensure passwords are at least 8 characters." - read -sp "Enter Container Password → " CONTAINER_PASSWORD - echo - read -sp "Confirm Container Password → " CONFIRM_PASSWORD - echo - PASSWORD_RETRIES=$(($PASSWORD_RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 4 - fi - done -else - CONFIRM_PASSWORD="$CONTAINER_PASSWORD" - while [[ "$CONFIRM_PASSWORD" != "$CONTAINER_PASSWORD" || ${#CONTAINER_PASSWORD} -lt 8 ]]; do - if [ $PASSWORD_RETRIES -gt 0 ]; then - echo "Sorry, try again. Ensure passwords are at least 8 characters." - read -sp "Enter Container Password → " CONTAINER_PASSWORD - echo - read -sp "Confirm Container Password → " CONFIRM_PASSWORD - echo - PASSWORD_RETRIES=$(($PASSWORD_RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 4 - fi - done -fi - # Choose Linux Distribution if [ -z "$LINUX_DISTRIBUTION" ]; then - echo "🐧 Available Linux Distributions:" - echo "1. Debian 12 (Bookworm)" - echo "2. Rocky 9 " - read -p "➡️ Choose a Linux Distribution (debian/rocky) → " LINUX_DISTRIBUTION + echo "🐧 Available Linux Distributions:" + echo "1. Debian 12 (Bookworm)" + echo "2. Rocky 9 " + read -p "➡️ Choose a Linux Distribution (debian/rocky) → " LINUX_DISTRIBUTION fi -while [ "${LINUX_DISTRIBUTION,,}" != "debian" ] && [ "${LINUX_DISTRIBUTION,,}" != "rocky" ]; do - echo "❌ Please choose a valid Linux Distribution." - echo "1. Debian 12 (Bookworm)" - echo "2. Rocky 9 " - read -p "➡️ Choose a Linux Distribution (debian/rocky) → " LINUX_DISTRIBUTION -done +if [ "${LINUX_DISTRIBUTION,,}" != "debian" ] && [ "${LINUX_DISTRIBUTION,,}" != "rocky" ]; then + LINUX_DISTRIBUTION="debian" +fi LINUX_DISTRIBUTION=${LINUX_DISTRIBUTION,,} @@ -135,53 +106,54 @@ PUB_FILE="key_$RANDOM_NUM.pub" TEMP_PUB_FILE="/root/bin/ssh/temp_pubs/$PUB_FILE" # in case two users are running this script at the same time, they do not overwrite each other's temp files touch "$TEMP_PUB_FILE" DETECT_PUBLIC_KEY=$(sudo /root/bin/ssh/detectPublicKey.sh "$SSH_KEY_FP" "$TEMP_PUB_FILE") -KEY_RETRIES=10 if [ "$DETECT_PUBLIC_KEY" == "Public key found for create-container" ]; then - echo "🔐 Public Key Found!" + echo "🔐 Public Key Found!" else - echo "🔍 Could not detect Public Key" - - if [ -z "$PUBLIC_KEY" ]; then - read -p "Enter Public Key (Allows Easy Access to Container) [OPTIONAL - LEAVE BLANK TO SKIP] → " PUBLIC_KEY - fi + echo "🔍 Could not detect Public Key" - # Check if key is valid - - while [[ "$PUBLIC_KEY" != "" && $(echo "$PUBLIC_KEY" | ssh-keygen -l -f - 2>&1 | tr -d '\r') == "(stdin) is not a public key file." ]]; do - if [ $KEY_RETRIES -gt 0 ]; then - echo "❌ \"$PUBLIC_KEY\" is not a valid key. Enter either a valid key or leave blank to skip." - read -p "Enter Public Key (Allows Easy Access to Container) [OPTIONAL - LEAVE BLANK TO SKIP] → " PUBLIC_KEY - KEY_RETRIES=$(($KEY_RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 5 - fi - done + if [ -z "$PUBLIC_KEY" ]; then + read -p "Enter Public Key (Allows Easy Access to Container) [OPTIONAL - LEAVE BLANK TO SKIP] → " PUBLIC_KEY + fi - if [ "$PUBLIC_KEY" != "" ]; then - echo "$PUBLIC_KEY" > "$AUTHORIZED_KEYS" && systemctl restart ssh - echo "$PUBLIC_KEY" > "$TEMP_PUB_FILE" - sudo /root/bin/ssh/publicKeyAppendJumpHost.sh "$PUBLIC_KEY" - fi + # Check if key is valid + + while [[ "$PUBLIC_KEY" != "" && $(echo "$PUBLIC_KEY" | ssh-keygen -l -f - 2>&1 | tr -d '\r') == "(stdin) is not a public key file." ]]; do + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Invalid Public Key" + writeLog "Invalid public key entered (GH_ACTION mode)" + exit 5 + fi + echo "❌ \"$PUBLIC_KEY\" is not a valid key. Enter either a valid key or leave blank to skip." + writeLog "Invalid public key entered: $PUBLIC_KEY" + read -p "Enter Public Key (Allows Easy Access to Container) [OPTIONAL - LEAVE BLANK TO SKIP] → " PUBLIC_KEY + done + + if [ "$PUBLIC_KEY" != "" ]; then + echo "$PUBLIC_KEY" > "$AUTHORIZED_KEYS" && systemctl restart ssh + echo "$PUBLIC_KEY" > "$TEMP_PUB_FILE" + sudo /root/bin/ssh/publicKeyAppendJumpHost.sh "$PUBLIC_KEY" + fi fi # Get HTTP Port Container Listens On -HTTP_PORT_RETRIES=10 if [ -z "$HTTP_PORT" ]; then read -p "Enter HTTP Port for your container to listen on (80-60000) → " HTTP_PORT + if [ "${GH_ACTION^^}" == "Y" ]; then + HTTP_PORT="3000" # Default to 3000 if not set + fi fi while ! [[ "$HTTP_PORT" =~ ^[0-9]+$ ]] || [ "$HTTP_PORT" -lt 80 ] || [ "$HTTP_PORT" -gt 60000 ]; do - if [ $HTTP_PORT_RETRIES -gt 0 ]; then - echo "❌ Invalid HTTP Port. It must be a number between 80 and 60,000." - read -p "Enter HTTP Port for your container to listen on (80-60000) → " HTTP_PORT - HTTP_PORT_RETRIES=$(($HTTP_PORT_RETRIES-1)) - else - echo "Too many incorrect attempts. Exiting..." - exit 6 - fi + if [ "${GH_ACTION^^}" == "Y" ]; then + outputError "Invalid HTTP Port. Must be between 80 and 60,000." + writeLog "Invalid HTTP port entered: $HTTP_PORT (GH_ACTION mode)" + exit 6 + fi + echo "❌ Invalid HTTP Port. It must be a number between 80 and 60,000." + writeLog "Invalid HTTP port entered: $HTTP_PORT" + read -p "Enter HTTP Port for your container to listen on (80-60000) → " HTTP_PORT done echo "✅ HTTP Port is set to $HTTP_PORT" @@ -207,12 +179,13 @@ while [ "${USE_OTHER_PROTOCOLS^^}" != "Y" ] && [ "${USE_OTHER_PROTOCOLS^^}" != " read -p "Does your Container require any protocols other than SSH and HTTP? (y/n) → " USE_OTHER_PROTOCOLS done -RANDOM_NUM=$(shuf -i 100000-999999 -n 1) -PROTOCOL_BASE_FILE="protocol_list_$RANDOM_NUM.txt" -PROTOCOL_FILE="/root/bin/protocols/$PROTOCOL_BASE_FILE" -touch "$PROTOCOL_FILE" - if [ "${USE_OTHER_PROTOCOLS^^}" == "Y" ]; then + + RANDOM_NUM=$(shuf -i 100000-999999 -n 1) + PROTOCOL_BASE_FILE="protocol_list_$RANDOM_NUM.txt" + PROTOCOL_FILE="/root/bin/protocols/$PROTOCOL_BASE_FILE" + touch "$PROTOCOL_FILE" + LIST_PROTOCOLS=() read -p "Enter the protocol abbreviation (e.g, LDAP for Lightweight Directory Access Protocol). Type \"e\" to exit → " PROTOCOL_NAME while [ "${PROTOCOL_NAME^^}" != "E" ]; do @@ -244,7 +217,6 @@ if [ "${USE_OTHER_PROTOCOLS^^}" == "Y" ]; then done fi - # Attempt to deploy application on start. if [ -z "$DEPLOY_ON_START" ]; then @@ -256,6 +228,12 @@ while [ "${DEPLOY_ON_START^^}" != "Y" ] && [ "${DEPLOY_ON_START^^}" != "N" ] && read -p "🚀 Do you want to deploy your project automatically? (y/n) → " DEPLOY_ON_START done +if [ "${GH_ACTION^^}" == "Y" ]; then + if [ ! -z "${RUNTIME_LANGUAGE^^}" ]; then + DEPLOY_ON_START="Y" + fi +fi + if [ "${DEPLOY_ON_START^^}" == "Y" ]; then source /root/bin/deploy-application.sh fi @@ -291,27 +269,38 @@ echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━ echo -e "${BOLD}${MAGENTA}🚀 Starting Container Creation...${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -ssh -t root@10.15.0.4 "bash -c '/var/lib/vz/snippets/clone-clxc.sh \ -'\''$CONTAINER_NAME'\'' \ -'\''$CONTAINER_PASSWORD'\'' \ -'\''$HTTP_PORT'\'' \ -'\''$PROXMOX_USERNAME'\'' \ -'\''$PUB_FILE'\'' \ -'\''$PROTOCOL_BASE_FILE'\'' \ -'\''$DEPLOY_ON_START'\'' \ -'\''$PROJECT_REPOSITORY'\'' \ -'\''$PROJECT_BRANCH'\'' \ -'\''$PROJECT_ROOT'\'' \ -'\''$INSTALL_COMMAND'\'' \ -'\''$BUILD_COMMAND'\'' \ -'\''$START_COMMAND'\'' \ -'\''$RUNTIME_LANGUAGE'\'' \ -'\''$ENV_FOLDER'\'' \ -'\''$SERVICES_FILE'\'' \ -'\''$LINUX_DISTRIBUTION'\'' \ -'\''$MULTI_COMPONENT'\'' -'\''$ROOT_START_COMMAND'\'' '" - +# Encode JSON variables +INSTALL_COMMAND_B64=$(echo -n "$INSTALL_COMMAND" | base64) +BUILD_COMMAND_B64=$(echo -n "$BUILD_COMMAND" | base64) +RUNTIME_LANGUAGE_B64=$(echo -n "$RUNTIME_LANGUAGE" | base64) +START_COMMAND_B64=$(echo -n "$START_COMMAND" | base64) + +REMOTE_CMD=( +/var/lib/vz/snippets/create-container.sh +"$CONTAINER_NAME" +"$GH_ACTION" +"$HTTP_PORT" +"$PROXMOX_USERNAME" +"$PUB_FILE" +"$PROTOCOL_BASE_FILE" +"$DEPLOY_ON_START" +"$PROJECT_REPOSITORY" +"$PROJECT_BRANCH" +"$PROJECT_ROOT" +"$INSTALL_COMMAND_B64" +"$BUILD_COMMAND_B64" +"$START_COMMAND_B64" +"$RUNTIME_LANGUAGE_B64" +"$ENV_FOLDER" +"$SERVICES_FILE" +"$LINUX_DISTRIBUTION" +"$MULTI_COMPONENT" +"$ROOT_START_COMMAND" +) + +QUOTED_REMOTE_CMD=$(printf ' %q' "${REMOTE_CMD[@]}") + +ssh -t root@10.15.0.4 "bash -c \"$QUOTED_REMOTE_CMD\"" rm -rf "$PROTOCOL_FILE" rm -rf "$TEMP_PUB_FILE" @@ -319,5 +308,5 @@ rm -rf "$TEMP_SERVICES_FILE_PATH" rm -rf "$ENV_FOLDER_PATH" unset CONFIRM_PASSWORD -unset CONTAINER_PASSWORD unset PUBLIC_KEY +unset PROXMOX_PASSWORD diff --git a/container creation/protocols/master_protocol_list.txt b/container creation/protocols/master_protocol_list.txt new file mode 100644 index 00000000..ba1d16e4 --- /dev/null +++ b/container creation/protocols/master_protocol_list.txt @@ -0,0 +1,145 @@ +TCPM 1 tcp +RJE 5 tcp +ECHO 7 tcp +DISCARD 9 tcp +DAYTIME 13 tcp +QOTD 17 tcp +MSP 18 tcp +CHARGEN 19 tcp +FTP 20 tcp +FTP 21 tcp +SSH 22 tcp +TELNET 23 tcp +SMTP 25 tcp +TIME 37 tcp +HNS 42 tcp +WHOIS 43 tcp +TACACS 49 tcp +DNS 53 tcp +BOOTPS 67 udp +BOOTPC 68 udp +TFTP 69 udp +GOPHER 70 tcp +FINGER 79 tcp +HTTP 80 tcp +KERBEROS 88 tcp +HNS 101 tcp +ISO-TSAP 102 tcp +POP2 109 tcp +POP3 110 tcp +RPC 111 tcp +AUTH 113 tcp +SFTP 115 tcp +UUCP-PATH 117 tcp +NNTP 119 tcp +NTP 123 udp +EPMAP 135 tcp +NETBIOS-NS 137 tcp +NETBIOS-DGM 138 udp +NETBIOS-SSN 139 tcp +IMAP 143 tcp +SQL-SRV 156 tcp +SNMP 161 udp +SNMPTRAP 162 udp +XDMCP 177 tcp +BGP 179 tcp +IRC 194 tcp +LDAP 389 tcp +NIP 396 tcp +HTTPS 443 tcp +SNPP 444 tcp +SMB 445 tcp +KPASSWD 464 tcp +SMTPS 465 tcp +ISAKMP 500 udp +EXEC 512 tcp +LOGIN 513 tcp +SYSLOG 514 udp +LPD 515 tcp +TALK 517 udp +NTALK 518 udp +RIP 520 udp +RIPNG 521 udp +RPC 530 tcp +UUCP 540 tcp +KLOGIN 543 tcp +KSHELL 544 tcp +DHCPV6-C 546 tcp +DHCPV6-S 547 tcp +AFP 548 tcp +RTSP 554 tcp +NNTPS 563 tcp +SUBMISSION 587 tcp +IPP 631 tcp +LDAPS 636 tcp +LDP 646 tcp +LINUX-HA 694 tcp +ISCSI 860 tcp +RSYNC 873 tcp +VMWARE 902 tcp +FTPS-DATA 989 tcp +FTPS 990 tcp +TELNETS 992 tcp +IMAPS 993 tcp +POP3S 995 tcp +SOCKS 1080 tcp +OPENVPN 1194 udp +OMGR 1311 tcp +MS-SQL-S 1433 tcp +MS-SQL-M 1434 udp +WINS 1512 tcp +ORACLE-SQL 1521 tcp +RADIUS 1645 tcp +RADIUS-ACCT 1646 tcp +L2TP 1701 udp +PPTP 1723 tcp +CISCO-ISL 1741 tcp +RADIUS 1812 udp +RADIUS-ACCT 1813 udp +NFS 2049 tcp +CPANEL 2082 tcp +CPANEL-SSL 2083 tcp +WHM 2086 tcp +WHM-SSL 2087 tcp +DA 2222 tcp +ORACLE-DB 2483 tcp +ORACLE-DBS 2484 tcp +XBOX 3074 tcp +HTTP-PROXY 3128 tcp +MYSQL 3306 tcp +RDP 3389 tcp +NDPS-PA 3396 tcp +SVN 3690 tcp +MSQL 4333 udp +METASPLOIT 4444 tcp +EMULE 4662 tcp +EMULE 4672 udp +RADMIN 4899 tcp +UPNP 5000 tcp +YMSG 5050 tcp +SIP 5060 tcp +SIP-TLS 5061 tcp +AIM 5190 tcp +XMPP-CLIENT 5222 tcp +XMPP-CLIENTS 5223 tcp +XMPP-SERVER 5269 tcp +POSTGRES 5432 tcp +VNC 5500 tcp +VNC-HTTP 5800 tcp +VNC 5900 tcp +X11 6000 tcp +BNET 6112 tcp +GNUTELLA 6346 tcp +SANE 6566 tcp +IRC 6667 tcp +IRCS 6697 tcp +BT 6881 tcp +HTTP-ALT 8000 tcp +HTTP-ALT 8008 tcp +HTTP-ALT 8080 tcp +HTTPS-ALT 8443 tcp +PDL-DS 9100 tcp +BACNET 9101 tcp +WEBMIN 10000 udp +MONGO 27017 tcp +TRACEROUTE 33434 udp \ No newline at end of file diff --git a/container registration/register-container.sh b/container creation/register-container.sh similarity index 100% rename from container registration/register-container.sh rename to container creation/register-container.sh diff --git a/container creation/services/service_map.json b/container creation/services/service_map_debian.json similarity index 56% rename from container creation/services/service_map.json rename to container creation/services/service_map_debian.json index 8f7618f4..2c99c524 100644 --- a/container creation/services/service_map.json +++ b/container creation/services/service_map_debian.json @@ -3,66 +3,66 @@ "curl https://install.meteor.com/ | sh" ], "mongodb": [ - "sudo apt-get update", - "sudo apt-get install -y gnupg curl", + "sudo apt update -y", + "sudo apt install -y gnupg curl", "curl -fsSL https://pgp.mongodb.com/server-7.0.asc | sudo gpg --dearmor -o /usr/share/keyrings/mongodb-server-7.0.gpg", "echo \"deb [ signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/debian bookworm/mongodb-org/7.0 main\" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list", - "sudo apt-get update", - "sudo apt-get install -y mongodb-org", + "sudo apt update -y", + "sudo apt install -y mongodb-org", "sudo systemctl enable mongod", "sudo systemctl start mongod" ], "redis": [ - "sudo apt-get update", - "sudo apt-get install -y redis-server", + "sudo apt update -y", + "sudo apt install -y redis-server", "sudo systemctl enable redis-server", "sudo systemctl start redis-server" ], "postgresql": [ - "sudo apt-get update", - "sudo apt-get install -y postgresql postgresql-contrib", + "sudo apt update -y", + "sudo apt install -y postgresql postgresql-contrib", "sudo systemctl enable postgresql", "sudo systemctl start postgresql" ], "apache": [ - "sudo apt-get update", - "sudo apt-get install -y apache2", + "sudo apt update -y", + "sudo apt install -y apache2", "sudo systemctl enable apache2", "sudo systemctl start apache2" ], "nginx": [ - "sudo apt-get update", - "sudo apt-get install -y nginx", + "sudo apt update -y", + "sudo apt install -y nginx", "sudo systemctl enable nginx", "sudo systemctl start nginx" ], "docker": [ - "sudo apt-get update", - "sudo apt-get install -y lsb-release", - "sudo apt-get install -y ca-certificates curl gnupg lsb-release", + "sudo apt update -y", + "sudo apt install -y lsb-release", + "sudo apt install -y ca-certificates curl gnupg lsb-release", "sudo install -m 0755 -d /etc/apt/keyrings", "curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg", - "echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable\" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null", - "sudo apt-get update", - "sudo apt-get install -y docker-ce docker-ce-cli containerd.io", + "echo 'deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian bookworm stable' | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null", + "sudo apt update -y", + "sudo apt install -y docker-ce docker-ce-cli containerd.io", "sudo systemctl enable docker", "sudo systemctl start docker" ], "rabbitmq": [ - "sudo apt-get update", - "sudo apt-get install -y rabbitmq-server", + "sudo apt update -y", + "sudo apt install -y rabbitmq-server", "sudo systemctl enable rabbitmq-server", "sudo systemctl start rabbitmq-server" ], "memcached": [ - "sudo apt-get update", - "sudo apt-get install -y memcached", + "sudo apt update -y", + "sudo apt install -y memcached", "sudo systemctl enable memcached", "sudo systemctl start memcached" ], "mariadb": [ - "sudo apt-get update", - "sudo apt-get install -y mariadb-server", + "sudo apt update -y", + "sudo apt install -y mariadb-server", "sudo systemctl enable mariadb", "sudo systemctl start mariadb" ] diff --git a/container creation/services/service_map_rocky.json b/container creation/services/service_map_rocky.json new file mode 100644 index 00000000..1b661433 --- /dev/null +++ b/container creation/services/service_map_rocky.json @@ -0,0 +1,99 @@ +{ + "meteor": [ + "dnf install tar -y", + "curl https://install.meteor.com/ | sh" + ], + "mongodb": [ + "sudo dnf install -y epel-release", + "sudo dnf update -y", + "sudo dnf install -y gnupg curl", + "curl -fsSL https://pgp.mongodb.com/server-7.0.asc | sudo gpg --dearmor -o /etc/pki/rpm-gpg/RPM-GPG-KEY-mongodb", + "echo '[mongodb-org-7.0]' | sudo tee /etc/yum.repos.d/mongodb-org-7.0.repo", + "echo 'name=MongoDB Repository' >> /etc/yum.repos.d/mongodb-org-7.0.repo", + "echo 'baseurl=https://repo.mongodb.org/yum/redhat/9/mongodb-org/7.0/x86_64/' >> /etc/yum.repos.d/mongodb-org-7.0.repo", + "echo 'gpgcheck=1' >> /etc/yum.repos.d/mongodb-org-7.0.repo", + "echo 'enabled=1' >> /etc/yum.repos.d/mongodb-org-7.0.repo", + "echo 'gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-mongodb' >> /etc/yum.repos.d/mongodb-org-7.0.repo", + "sudo dnf install -y mongodb-org", + "sudo systemctl enable mongod", + "sudo systemctl start mongod" + ], + "redis": [ + "sudo dnf install -y epel-release", + "sudo dnf update -y", + "sudo dnf install -y redis", + "sudo systemctl enable redis", + "sudo systemctl start redis" + ], + "postgresql": [ + "sudo dnf install -y epel-release", + "sudo dnf update -y", + "sudo dnf install -y postgresql-server postgresql-contrib", + "sudo postgresql-setup --initdb", + "sudo systemctl enable postgresql", + "sudo systemctl start postgresql" + ], + "apache": [ + "sudo dnf install -y epel-release", + "sudo dnf update -y", + "sudo dnf install -y httpd", + "sudo systemctl enable httpd", + "sudo systemctl start httpd" + ], + "httpd": [ + "sudo dnf install -y epel-release", + "sudo dnf update -y", + "sudo dnf install -y httpd", + "sudo systemctl enable httpd", + "sudo systemctl start httpd" + ], + "nginx": [ + "sudo dnf install -y epel-release", + "sudo dnf update -y", + "sudo dnf install -y nginx", + "sudo systemctl enable nginx", + "sudo systemctl start nginx" + ], + "docker": [ + "sudo dnf update -y", + "sudo dnf install -y yum-utils device-mapper-persistent-data lvm2", + "sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo", + "sudo dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin", + "sudo systemctl enable docker", + "sudo systemctl start docker" + ], + "rabbitmq": [ + "sudo dnf install -y epel-release", + "sudo dnf install -y erlang", + "sudo dnf update -y", + "rpm --import 'https://github.com/rabbitmq/signing-keys/releases/download/3.0/rabbitmq-release-signing-key.asc'", + "rpm --import 'https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-server.9F4587F226208342.key'", + "echo '[rabbitmq-server]' | sudo tee /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo", + "echo 'name=rabbitmq-server' | sudo tee -a /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo", + "echo 'baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/9/$basearch' | sudo tee /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo", + "echo 'repo_gpgcheck=1' | sudo tee -a /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo", + "echo 'gpgcheck=1' | sudo tee -a /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo", + "echo 'enabled=1' | sudo tee -a /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo", + "echo 'gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey' | sudo tee -a /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo", + "echo 'sslverify=1' | sudo tee -a /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo", + "echo 'sslcacert=/etc/pki/tls/certs/ca-bundle.crt' | sudo tee -a /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo", + "echo 'metadata_expire=300' | sudo tee -a /etc/yum.repos.d/rabbitmq_rabbitmq-server.repo", + "sudo dnf install -y rabbitmq-server", + "sudo systemctl enable rabbitmq-server", + "sudo systemctl start rabbitmq-server" + ], + "memcached": [ + "sudo dnf install -y epel-release", + "sudo dnf update -y", + "sudo dnf install -y memcached", + "sudo systemctl enable memcached", + "sudo systemctl start memcached" + ], + "mariadb": [ + "sudo dnf install -y epel-release", + "sudo dnf update -y", + "sudo dnf install -y mariadb-server", + "sudo systemctl enable mariadb", + "sudo systemctl start mariadb" + ] +} \ No newline at end of file diff --git a/container creation/setup-runner.sh b/container creation/setup-runner.sh index 53bcf442..382f4f87 100644 --- a/container creation/setup-runner.sh +++ b/container creation/setup-runner.sh @@ -1,8 +1,16 @@ #!/bin/bash # A script for cloning a Distro template, installing, and starting a runner on it. -# Last Modified by Maxwell Klema on July 20th, 2025 +# Last Modified by Maxwell Klema on August 5th, 2025 # ------------------------------------------------ +outputError() { + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}${MAGENTA}❌ Script Failed. Exiting... ${RESET}" + echo -e "$2" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + exit $1 +} + BOLD='\033[1m' RESET='\033[0m' @@ -15,10 +23,8 @@ echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━ source /var/lib/vz/snippets/helper-scripts/PVE_user_authentication.sh #Authenticate User source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh #Ensure container does not exist. -CONTAINER_EXISTS=$? - -if [ "$CONTAINER_EXISTS" != 1 ]; then - exit $CONTAINER_EXISTS; # Container is not free to user, either someone else owns it or the user owns it. +if [ ! -z "$CONTAINER_OWNERSHIP" ]; then + outputError 1 "You already own a container with name \"$CONTAINER_NAME\". Please delete it before creating a new one." fi # Cloning Container Template and Setting it up ===== @@ -30,7 +36,7 @@ TEMPLATE_NAME="template-$REPO_BASE_NAME-$REPO_BASE_NAME_WITH_OWNER" CTID_TEMPLATE=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$TEMPLATE_NAME" '$3 == name {print $1}') case "${LINUX_DISTRIBUTION^^}" in - DEBIAN) PACKAGE_MANAGER="apt-get" ;; + "") PACKAGE_MANAGER="apt-get" ;; ROCKY) PACKAGE_MANAGER="dnf" ;; esac @@ -38,10 +44,14 @@ esac if [ -z "$CTID_TEMPLATE" ]; then case "${LINUX_DISTRIBUTION^^}" in - DEBIAN) CTID_TEMPLATE="160" ;; + "") CTID_TEMPLATE="160" ;; ROCKY) CTID_TEMPLATE="138" ;; esac -fi +fi + +if [ "${LINUX_DISTRIBUTION^^}" != "ROCKY" ]; then + LINUX_DISTRIBUTION="DEBIAN" +fi REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") REPO_BASE_NAME_WITH_OWNER=$(echo "$PROJECT_REPOSITORY" | cut -d'/' -f4) @@ -58,7 +68,10 @@ pct clone $CTID_TEMPLATE $NEXT_ID \ echo "⏳ Setting Container Properties..." pct set $NEXT_ID \ --tags "$PROXMOX_USERNAME" \ - --onboot 1 > /dev/null 2>&1 + --tags "$LINUX_DISTRIBUTION" \ + --onboot 1 \ + --cores 4 \ + --memory 4096 > /dev/null 2>&1 pct start $NEXT_ID > /dev/null 2>&1 pveum aclmod /vms/$NEXT_ID --user "$PROXMOX_USERNAME@pve" --role PVEVMUser > /dev/null 2>&1 @@ -67,23 +80,29 @@ sleep 5 echo "⏳ DHCP Allocating IP Address..." CONTAINER_IP=$(pct exec $NEXT_ID -- hostname -I | awk '{print $1}') -# Set password inside the container -pct exec $NEXT_ID -- bash -c "echo 'root:$CONTAINER_PASSWORD' | chpasswd" > /dev/null 2>&1 - # Setting Up Github Runner ===== # Get Temporary Token echo "🪙 Getting Authentication Token..." -AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/registration-token --header "Authorization: token $GITHUB_PAT") -TOKEN=$(echo "$AUTH_TOKEN_RESPONSE" | jq -r '.token') +AUTH_TOKEN_RESPONSE=$(curl --location --request POST https://api.github.com/repos/$REPO_BASE_NAME_WITH_OWNER/$REPO_BASE_NAME/actions/runners/registration-token --header "Authorization: token $GITHUB_PAT" --write-out "HTTPSTATUS:%{http_code}" --silent) + +HTTP_STATUS=$(echo "$AUTH_TOKEN_RESPONSE" | grep -o "HTTPSTATUS:[0-9]*" | cut -d: -f2) +AUTH_TOKEN_BODY=$(echo "$AUTH_TOKEN_RESPONSE" | sed 's/HTTPSTATUS:[0-9]*$//') + +if [ "$HTTP_STATUS" != "201" ]; then + outputError 1 "Failed to get GitHub authentication token. HTTP Status: $HTTP_STATUS\nResponse: $AUTH_TOKEN_BODY" +fi + +TOKEN=$(echo "$AUTH_TOKEN_BODY" | jq -r '.token') -pct enter $NEXT_ID < /dev/null -rm -rf /root/container-updates.log | true && \ +pct enter $NEXT_ID < /dev/null 2>&1 +rm -rf /root/container-updates.log || true && \ cd /actions-runner && export RUNNER_ALLOW_RUNASROOT=1 && \ -export runProcess=\$(ps aux | grep run.sh | awk '{print \$2}' | head -n 1) && kill -9 \$runProcess || true && \ -rm -rf .runner .credentials && rm -rf _work/* /var/log/runner/* && \ +runProcess=\$(ps aux | grep "[r]un.sh" | awk '{print \$2}' | head -n 1) && \ +if [ ! -z "\$runProcess" ]; then kill -9 \$runProcess || true; fi && \ +rm -rf .runner .credentials && rm -rf _work/* /var/log/runner/* 2>/dev/null || true && \ export RUNNER_ALLOW_RUNASROOT=1 && \ -./config.sh --url $PROJECT_REPOSITORY --token $TOKEN --labels $CONTAINER_NAME --name $CONTAINER_NAME +./config.sh --url $PROJECT_REPOSITORY --token $TOKEN --labels $CONTAINER_NAME --name $CONTAINER_NAME --unattended EOF # Generate RSA Keys ===== diff --git a/container maintenance/start_services.sh b/container creation/start_services.sh similarity index 59% rename from container maintenance/start_services.sh rename to container creation/start_services.sh index 165a96a0..7942ad02 100644 --- a/container maintenance/start_services.sh +++ b/container creation/start_services.sh @@ -1,10 +1,9 @@ #!/bin/bash # Script ran by a virtual terminal session to start services and migrate a container # Script is only ran on GH action workflows when runner disconnects -# Last Modified by Maxwell Klema on July 23rd, 2025 +# Last Modified by Maxwell Klema on August 5th, 2025 # ------------------------------------------------ -set -x CONTAINER_ID="$1" CONTAINER_NAME="$2" REPO_BASE_NAME="$3" @@ -20,28 +19,27 @@ BUILD_COMMAND=$(echo "${12}" | base64 -d) RUNTIME_LANGUAGE=$(echo "${13}" | base64 -d) GH_ACTION="${14}" PROJECT_BRANCH="${15}" -GITHUB_PAT="${16}" -UPDATE_CONTAINER="${17}" +UPDATE_CONTAINER="${16}" CONTAINER_NAME="${CONTAINER_NAME,,}" -sleep 3 -pct stop $CONTAINER_ID > /dev/null 2>&1 - -echo "$START_COMMAND" -echo "$BUILD_COMMAND" -echo "$RUNTIME_LANGUAGE" - -sleep 10 +if [ "${GH_ACTION^^}" == "Y" ]; then + sleep 8 # Wait for Job to Complete +fi +if (( $CONTAINER_ID % 2 == 0 )) && [ "$UPDATE_CONTAINER" == "true" ]; then + ssh root@10.15.0.5 "pct stop $CONTAINER_ID" > /dev/null 2>&1 +else + pct stop $CONTAINER_ID > /dev/null 2>&1 +fi # Create template if on default branch ==== -# source /var/lib/vz/snippets/helper-scripts/create-template.sh +source /var/lib/vz/snippets/helper-scripts/create-template.sh if (( $CONTAINER_ID % 2 == 0 )); then if [ "$UPDATE_CONTAINER" != "true" ]; then pct migrate $CONTAINER_ID intern-phxdc-pve2 --target-storage containers-pve2 --online > /dev/null 2>&1 - sleep 40 # wait for migration to finish (fix this later) + sleep 5 # wait for migration to finish (fix this later) fi ssh root@10.15.0.5 "pct start $CONTAINER_ID" @@ -59,18 +57,14 @@ if (( $CONTAINER_ID % 2 == 0 )); then START_CMD="$3" COMP_DIR="$4" + if [ -z "$BUILD_CMD" ]; then + BUILD_CMD="true" + fi + if [ "${RUNTIME^^}" == "NODEJS" ]; then - if [ "$BUILD_CMD" == "" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 - else - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'export PATH=\$PATH:/usr/local/bin && $BUILD_CMD && pm2 start bash -- -c \"cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD\"'" > /dev/null 2>&1 - fi + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c \"mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp tmux new-session -d 'export HOME=/root export PATH=\\\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 elif [ "${RUNTIME^^}" == "PYTHON" ]; then - if [ "$BUILD_CMD" == "" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'\"" > /dev/null 2>&1 - else - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- script -q -c \"tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 - fi + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c \"mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp tmux new-session -d 'export HOME=/root export PATH=\\\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 fi } @@ -93,11 +87,11 @@ if (( $CONTAINER_ID % 2 == 0 )); then startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." fi fi - ssh root@10.15.0.5 "pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2" # PVE 1 else - pct start $CONTAINER_ID | true + pct start $CONTAINER_ID || true + sleep 5 if [ "${GH_ACTION^^}" == "Y" ]; then pct exec $CONTAINER_ID -- bash -c "systemctl start github-runner" fi @@ -109,21 +103,14 @@ else START_CMD="$3" COMP_DIR="$4" + if [ -z "$BUILD_CMD" ]; then + BUILD_CMD="true" + fi + if [ "${RUNTIME^^}" == "NODEJS" ]; then - if [ "$BUILD_CMD" == "" ]; then - pct exec $CONTAINER_ID -- bash -c "export PATH=\$PATH:/usr/local/bin && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD'" > /dev/null 2>&1 - else - pct enter $CONTAINER_ID < /dev/null -export PATH=\$PATH:/usr/local/bin && \ -$BUILD_CMD || true && pm2 start bash -- -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $START_CMD' -EOF - fi + pct exec "$CONTAINER_ID" -- bash -c "mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp/tmux-0 tmux new-session -d \"export HOME=/root && export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $BUILD_CMD && $START_CMD\"" elif [ "${RUNTIME^^}" == "PYTHON" ]; then - if [ "$BUILD_CMD" == "" ]; then - pct exec $CONTAINER_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $START_CMD'" > /dev/null 2>&1 - else - pct exec $CONTAINER_ID -- script -q -c "tmux new-session -d 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'" > /dev/null 2>&1 - fi + pct exec "$CONTAINER_ID" -- bash -c "mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp/tmux-0 tmux new-session -d \"export HOME=/root &&export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $BUILD_CMD && $START_CMD\"" fi } @@ -145,5 +132,4 @@ EOF else startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." fi - pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 > /dev/null -fi \ No newline at end of file +fi diff --git a/container registration/register_proxy_hook.sh b/container registration/register_proxy_hook.sh deleted file mode 100644 index a4ade333..00000000 --- a/container registration/register_proxy_hook.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# /var/lib/vz/snippets/register_proxy_hook.sh - -echo "DEBUG: Hook script /var/lib/vz/snippets/register_proxy_hook.sh started. Event: $2, CTID: $1" >> /tmp/hook_debug.log - -# Hook script for container events -case "$2" in - post-start) - echo "DEBUG: Calling register-container.sh for CTID: $1" >> /tmp/hook_debug.log - /var/lib/vz/snippets/register-container.sh "$1" >> /tmp/hook_debug.log 2>&1 - echo "DEBUG: register-container.sh finished." >> /tmp/hook_debug.log - ;; - *) - echo "DEBUG: Unhandled hook event: $2 for CTID: $1" >> /tmp/hook_debug.log - ;; -esac -echo "DEBUG: Hook script /var/lib/vz/snippets/register_proxy_hook.sh finished." >> /tmp/hook_debug.log \ No newline at end of file diff --git a/dnsmasq service/README.md b/dnsmasq service/README.md new file mode 100644 index 00000000..28633abd --- /dev/null +++ b/dnsmasq service/README.md @@ -0,0 +1 @@ +# DNS Server \ No newline at end of file diff --git a/gateway/README.md b/gateway/README.md new file mode 100644 index 00000000..3fb79328 --- /dev/null +++ b/gateway/README.md @@ -0,0 +1 @@ +# Gateway (Intern-phxdc-pve1) \ No newline at end of file diff --git a/jump server/extract-fingerprint.sh b/gateway/extract-fingerprint.sh similarity index 100% rename from jump server/extract-fingerprint.sh rename to gateway/extract-fingerprint.sh diff --git a/gateway/prune_iptables.sh b/gateway/prune_iptables.sh new file mode 100644 index 00000000..abc1aec2 --- /dev/null +++ b/gateway/prune_iptables.sh @@ -0,0 +1,173 @@ +#!/bin/bash + +# Script to prune iptables rules for containers that no longer exist +# Author: Carter Myers + +# Enable strict mode: +# -e: Exit immediately if a command exits with a non-zero status. +# -u: Treat unset variables as an error when substituting. +# -o pipefail: The return value of a pipeline is the status of the last command +# to exit with a non-zero status, or zero if all commands exit successfully. +set -euo pipefail + +# --- Configuration --- +REMOTE_HOST="intern-nginx" +REMOTE_FILE="/etc/nginx/port_map.json" +LOCAL_FILE="/tmp/port_map.json" +LOG_FILE="/var/log/prune_iptables.log" +PVE_NODES=("localhost" "10.15.0.5") + +# Function to log messages with a timestamp +log_message() { + echo "[$(date)] $1" >> "$LOG_FILE" +} + +# --- 1. Fetch port_map.json from remote host --- +log_message "Fetching port_map.json from $REMOTE_HOST..." +if ! scp "$REMOTE_HOST:$REMOTE_FILE" "$LOCAL_FILE" >/dev/null 2>&1; then + log_message "ERROR: Could not fetch $REMOTE_FILE from $REMOTE_HOST" + exit 1 +fi +log_message "Successfully fetched $REMOTE_FILE to $LOCAL_FILE." + +# --- 2. Build list of existing hostnames --- +EXISTING_HOSTNAMES="" +for node in "${PVE_NODES[@]}"; do + log_message "Checking containers on $node..." + if [[ "$node" == "localhost" ]]; then + CTIDS=$(pct list | awk 'NR>1 {print $1}' || true) + log_message "DEBUG: Local CTIDs: [${CTIDS:-}]" + for id in $CTIDS; do + hn=$(pct config "$id" 2>/dev/null | grep -i '^hostname:' | awk '{print $2}' | tr -d '[:space:]' || true) + [[ -n "$hn" ]] && EXISTING_HOSTNAMES+="$hn"$'\n' + done + else + log_message "DEBUG: Checking remote node: $node" + CTIDS_CMD="pct list | awk 'NR>1 {print \$1}'" + CTIDS_OUTPUT=$(ssh "$node" "$CTIDS_CMD" 2>&1 || true) + if [[ "$CTIDS_OUTPUT" =~ "Permission denied" || "$CTIDS_OUTPUT" =~ "Connection refused" || "$CTIDS_OUTPUT" =~ "Host key verification failed" ]]; then + log_message "ERROR: SSH to $node failed: $CTIDS_OUTPUT" + continue + fi + log_message "DEBUG: CTIDs on $node: [${CTIDS_OUTPUT:-}]" + for id in $CTIDS_OUTPUT; do + HN_CMD="pct config $id 2>/dev/null | grep -i '^hostname:' | awk '{print \$2}'" + HN_OUTPUT=$(ssh "$node" "$HN_CMD" 2>&1 || true) + if [[ "$HN_OUTPUT" =~ "Permission denied" || "$HN_OUTPUT" =~ "No such file" ]]; then + log_message "ERROR: Failed to get hostname for $id on $node: $HN_OUTPUT" + continue + fi + hn=$(echo "$HN_OUTPUT" | tr -d '[:space:]') + [[ -n "$hn" ]] && EXISTING_HOSTNAMES+="$hn"$'\n' + done + fi +done + +# Remove any empty lines from EXISTING_HOSTNAMES +EXISTING_HOSTNAMES=$(echo "$EXISTING_HOSTNAMES" | sed '/^$/d') +log_message "Existing hostnames collected:" +log_message "$EXISTING_HOSTNAMES" + +# --- 3. Prune iptables and port_map.json --- +log_message "Pruning iptables and port_map.json..." +cp "$LOCAL_FILE" "$LOCAL_FILE.bak" +log_message "Created backup of $LOCAL_FILE at $LOCAL_FILE.bak" + +HOSTNAMES_IN_JSON=$(jq -r 'keys[]' "$LOCAL_FILE") +mapfile -t EXISTING_ARRAY <<< "$EXISTING_HOSTNAMES" + +# Helper function to check if a hostname exists in the collected list +hostname_exists() { + local h=$(echo "$1" | tr -d '[:space:]') + for existing in "${EXISTING_ARRAY[@]}"; do + if [[ "${h,,}" == "${existing,,}" ]]; then # Case-insensitive comparison + return 0 + fi + done + return 1 +} + +for hostname in $HOSTNAMES_IN_JSON; do + trimmed_hostname=$(echo "$hostname" | tr -d '[:space:]') + if hostname_exists "$trimmed_hostname"; then + log_message "Keeping entry: $trimmed_hostname" + else + ip=$(jq -r --arg h "$hostname" '.[$h].ip // "unknown"' "$LOCAL_FILE") + ports=$(jq -c --arg h "$hostname" '.[$h].ports // {}' "$LOCAL_FILE") + log_message "Stale entry detected: $hostname (IP: $ip, Ports: $ports) - removing..." + + # --- IPTABLES REMOVAL --- + # Capture rules into an array first to avoid subshell issues with 'while read' + mapfile -t RULES_TO_DELETE < <(sudo iptables -t nat -S | grep -w "$ip" || true) # Added sudo, || true to prevent pipefail if grep finds nothing + + if [[ ${#RULES_TO_DELETE[@]} -gt 0 ]]; then + log_message "Found ${#RULES_TO_DELETE[@]} iptables rules for $hostname. Attempting removal..." + for rule in "${RULES_TO_DELETE[@]}"; do + cleaned_rule=$(echo "$rule" | sed 's/^-A /-D /') + log_message "Attempting to remove iptables rule: sudo iptables -t nat $cleaned_rule" + if sudo iptables -t nat $cleaned_rule; then + log_message "Removed iptables rule: $cleaned_rule" + else + log_message "ERROR: Failed to remove iptables rule: $cleaned_rule (Exit status: $?)" + fi + done + else + log_message "No iptables rules found for $hostname to remove." + fi + + # --- JSON ENTRY REMOVAL --- + log_message "Attempting to remove $hostname from local port_map.json..." + if jq "del(.\"$hostname\")" "$LOCAL_FILE" > "${LOCAL_FILE}.tmp"; then + if mv "${LOCAL_FILE}.tmp" "$LOCAL_FILE"; then + log_message "Successfully removed $hostname from local port_map.json." + else + log_message "ERROR: Failed to move temporary file to $LOCAL_FILE for $hostname." + exit 1 # Critical failure, exit + fi + else + log_message "ERROR: jq failed to delete $hostname from $LOCAL_FILE." + exit 1 # Critical failure, exit + fi + + # Confirm deletion from local file + if jq -e --arg h "$hostname" 'has($h)' "$LOCAL_FILE" >/dev/null; then + log_message "ERROR: $hostname still exists in local port_map.json after deletion attempt!" + else + log_message "Confirmed $hostname removed from local port_map.json." + fi + fi +done + +# --- 4. Upload and verify updated file on remote --- +log_message "Uploading updated port_map.json to $REMOTE_HOST..." +TEMP_REMOTE="/tmp/port_map.json" + +if scp "$LOCAL_FILE" "$REMOTE_HOST:$TEMP_REMOTE" >/dev/null 2>&1; then + log_message "Uploaded to $REMOTE_HOST:$TEMP_REMOTE" +else + log_message "ERROR: Failed to upload $TEMP_REMOTE to $REMOTE_HOST" + exit 1 +fi + +# Check if deleted hostnames still exist in uploaded file +log_message "Verifying remote file content..." +for hostname in $HOSTNAMES_IN_JSON; do + if ! hostname_exists "$hostname"; then # Only check for hostnames that *should* have been deleted + if ssh "$REMOTE_HOST" "grep -q '\"$hostname\"' $TEMP_REMOTE"; then + log_message "WARNING: $hostname still exists in uploaded $TEMP_REMOTE on $REMOTE_HOST!" + else + log_message "Verified $hostname was removed in uploaded file on $REMOTE_HOST." + fi + fi +done + +# Move uploaded file into place on the remote host +log_message "Moving uploaded file into final position on $REMOTE_HOST..." +if ssh "$REMOTE_HOST" "sudo cp $TEMP_REMOTE $REMOTE_FILE && sudo chown root:root $REMOTE_FILE && sudo chmod 644 $REMOTE_FILE && rm $TEMP_REMOTE"; then + log_message "Copied updated port_map.json to $REMOTE_FILE on $REMOTE_HOST" +else + log_message "ERROR: Failed to replace $REMOTE_FILE on $REMOTE_HOST" + exit 1 +fi + +log_message "Prune complete." \ No newline at end of file diff --git a/gateway/prune_temp_files.sh b/gateway/prune_temp_files.sh new file mode 100644 index 00000000..1b171fd1 --- /dev/null +++ b/gateway/prune_temp_files.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Script to prune all temporary files (env vars, protocols, services, and public keys) +# Last Updated July 28th 2025 Maxwell Klema + +LOG_FILE="/var/log/pruneTempFiles.log" + +writeLog() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')]: $1" >> "$LOG_FILE" +} + +# Function to remove temporary environment variable Folders +removeTempEnvVars() { + TEMP_ENV_FOLDER="/var/lib/vz/snippets/container-env-vars" + while read -r line; do + if [[ "$line" == /var/lib/vz/snippets/container-env-vars/env_* ]]; then + rm -rf "$line" > /dev/null 2>&1 + writeLog "Removed temporary environment variable folder: $line" + fi + done < <(find "$TEMP_ENV_FOLDER" -maxdepth 1 -type d -name "env_*") +} + +# Function to remove temporary services file +removeTempServices() { + TEMP_SERVICES_FOLDER="/var/lib/vz/snippets/container-services" + while read -r line; do + if [[ "$line" == /var/lib/vz/snippets/container-services/services_* ]]; then + rm -f "$line" + writeLog "Removed temporary services file: $line" + fi + done < <(find "$TEMP_SERVICES_FOLDER" -maxdepth 1 -type f -name "services_*") +} + +# Function to remove temporary public key files +removeTempPublicKeys() { + TEMP_PUB_FOLDER="/var/lib/vz/snippets/container-public-keys" + while read -r line; do + if [[ "$line" == /var/lib/vz/snippets/container-public-keys/key_* ]]; then + rm -f "$line" + writeLog "Removed temporary public key file: $line" + fi + done < <(find "$TEMP_PUB_FOLDER" -maxdepth 1 -type f -name "key_*") +} + +# Function to remove temporary protocol files +removeTempProtocols() { + TEMP_PROTOCOL_FOLDER="/var/lib/vz/snippets/container-port-maps" + while read -r line; do + if [[ "$line" == /var/lib/vz/snippets/container-port-maps/protocol_list* ]]; then + rm -f "$line" + writeLog "Removed temporary protocol file: $line" + fi + done < <(find "$TEMP_PROTOCOL_FOLDER" -maxdepth 1 -type f -name "protocol_list*") +} + +# Main function to prune all temporary files +pruneTempFiles() { + writeLog "Starting to prune temporary files..." + removeTempEnvVars + removeTempServices + removeTempPublicKeys + removeTempProtocols + writeLog "Finished pruning temporary files." +} + +# Execute the main function +pruneTempFiles +exit 0 \ No newline at end of file diff --git a/nginx reverse proxy/README.md b/nginx reverse proxy/README.md new file mode 100644 index 00000000..f28dcebe --- /dev/null +++ b/nginx reverse proxy/README.md @@ -0,0 +1 @@ +# Nginx Reverse Proxy \ No newline at end of file diff --git a/nginx proxy/nginx.conf b/nginx reverse proxy/nginx.conf similarity index 100% rename from nginx proxy/nginx.conf rename to nginx reverse proxy/nginx.conf diff --git a/nginx proxy/port_map.js b/nginx reverse proxy/port_map.js similarity index 100% rename from nginx proxy/port_map.js rename to nginx reverse proxy/port_map.js diff --git a/nginx proxy/reverse_proxy.conf b/nginx reverse proxy/reverse_proxy.conf similarity index 100% rename from nginx proxy/reverse_proxy.conf rename to nginx reverse proxy/reverse_proxy.conf diff --git a/proxmox-launchpad b/proxmox-launchpad new file mode 160000 index 00000000..038aff5a --- /dev/null +++ b/proxmox-launchpad @@ -0,0 +1 @@ +Subproject commit 038aff5ad0eacd9f77935ae8819ab59da13fc981 From d8be8156da80576c29e609576f693a9234d31001 Mon Sep 17 00:00:00 2001 From: Maxwell Klema <80615123+maxklema@users.noreply.github.com> Date: Tue, 5 Aug 2025 16:40:35 -0400 Subject: [PATCH 23/25] Re organization (#3) * LDAP configuration and prune scripts * proxmox deployment changes * updated container-creation scripts + re-organization * READMEs in each directory, re-organization, updated ci-cd files * READMEs in each directory, re-organization, updated ci-cd files * proxmox launchpad submodule in ci-cd automation * proxmox launchpad submodule * proxmox launchpad submodule * Updated Root README + LDAP Folder * UPDATED readme --- .gitmodules | 11 ++- LDAP/LDAPServer | 1 + LDAP/README.md | 1 + LDAP/pown | 1 + README.md | 153 +++++++++++++++++++++-------- ci-cd automation/Proxmox-Launchpad | 1 + 6 files changed, 122 insertions(+), 46 deletions(-) create mode 160000 LDAP/LDAPServer create mode 100644 LDAP/README.md create mode 160000 LDAP/pown create mode 160000 ci-cd automation/Proxmox-Launchpad diff --git a/.gitmodules b/.gitmodules index c8f2499b..a38adba7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,8 @@ -[submodule "proxmox-launchpad"] - path = proxmox-launchpad - url = https://github.com/maxklema/proxmox-launchpad.git +[submodule "LDAP/pown"] + path = LDAP/pown + url = https://github.com/anishapant21/pown.sh.git +[submodule "LDAP/LDAPServer"] + path = LDAP/LDAPServer + url = https://github.com/mieweb/LDAPServer.git +[submodule "ci-cd automation/Proxmox-Launchpad"] + path = ci-cd automation/Proxmox-Launchpad diff --git a/LDAP/LDAPServer b/LDAP/LDAPServer new file mode 160000 index 00000000..00b8dffc --- /dev/null +++ b/LDAP/LDAPServer @@ -0,0 +1 @@ +Subproject commit 00b8dffc95e015b4f9849a012afd1f1d7369aff1 diff --git a/LDAP/README.md b/LDAP/README.md new file mode 100644 index 00000000..f4d89aa8 --- /dev/null +++ b/LDAP/README.md @@ -0,0 +1 @@ +## LDAP \ No newline at end of file diff --git a/LDAP/pown b/LDAP/pown new file mode 160000 index 00000000..a2a19a50 --- /dev/null +++ b/LDAP/pown @@ -0,0 +1 @@ +Subproject commit a2a19a5005349dba9d5d39200c9552c41c985e20 diff --git a/README.md b/README.md index 51a226b4..282b8e82 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,132 @@ # opensource-mieweb -Configuration storage for the [opensource.mieweb.com](https://opensource.mieweb.com) Proxmox project. +Configuration storage for the [opensource.mieweb.org](https://opensource.mieweb.org:8006) Proxmox project. -This repository contains configuration files and scripts for managing a Proxmox-based container hosting environment, including automated DNS, NGINX reverse proxy, and dynamic port mapping. +This repository contains configuration files and scripts for managing a Proxmox-based container hosting environment, including automated DNS, NGINX reverse proxy, dynamic port mapping, and the Proxmox LaunchPad GitHub Action for automated container deployment. +## Cluster Graph + +```mermaid + +graph TD + %% Repository Structure + REPO[opensource-mieweb Repository] + + %% All Main Folders + REPO --> CICD[ci-cd automation] + REPO --> CC[container creation] + REPO --> DNS[dnsmasq service] + REPO --> GW[gateway] + REPO --> LDAP[LDAP] + REPO --> NGINX[nginx reverse proxy] + REPO --> PL[proxmox-launchpad] + + %% Core Workflow Connections + CC --> |creates| CONTAINER[LXC Container] + CONTAINER --> |Updates Container Map| NGINX + CONTAINER --> |updates| DNS + CONTAINER --> | Updates IP Tables| GW + CONTAINER --> |authenticates via| LDAP + + %% CI/CD Operations + CICD --> |manages| CONTAINER + PL --> |automates| CC + PL --> |uses| CICD + + %% User Access Flow + USER[User Access] --> DNS + DNS --> NGINX + NGINX --> CONTAINER + + %% Styling + classDef folder fill:#e3f2fd,stroke:#1976d2,stroke-width:2px + classDef system fill:#f1f8e9,stroke:#689f38,stroke-width:2px + classDef user fill:#fff3e0,stroke:#f57c00,stroke-width:2px + + class CICD,CC,DNS,GW,LDAP,NGINX,PL folder + class CONTAINER system + class USER user ``` -│ README.md -│ -├───intern-dnsmasq -│ dnsmasq.conf -│ -├───intern-nginx -│ nginx.conf -│ port_map.js -│ reverse_proxy.conf -│ -└───intern-phxdc-pve1 - register-container.sh - register_proxy_hook.sh -``` -## Repository Structure -- [`intern-dnsmasq/dnsmasq.conf`](intern-dnsmasq/dnsmasq.conf): - Dnsmasq configuration for DHCP and DNS, including wildcard routing for the reverse proxy. +### Core Infrastructure + +- [`dnsmasq service/`](dnsmasq%20service/): + Contains Dnsmasq configuration for DHCP and DNS services, including wildcard routing for the reverse proxy and container network management. + +- [`nginx reverse proxy/`](nginx%20reverse%20proxy/): + Houses NGINX configuration files for the reverse proxy setup, including JavaScript modules for dynamic backend resolution and SSL certificate management. -- [`intern-nginx/nginx.conf`](intern-nginx/nginx.conf): - Main NGINX configuration, loading the JavaScript module for dynamic backend resolution. +- [`gateway/`](gateway/): + Gateway configuration and management scripts for network routing and access control between the internal container network and external traffic. Also contains daily clean up scripts for the cluster. -- [`intern-nginx/reverse_proxy.conf`](intern-nginx/reverse_proxy.conf): - NGINX reverse proxy config using dynamic JavaScript lookups for backend containers. +### Container Management -- [`intern-nginx/port_map.js`](intern-nginx/port_map.js): - JavaScript module for NGINX to map subdomains to backend container IPs and ports using a JSON file. +- [`container creation/`](container%20creation/): + Contains comprehensive scripts for LXC container lifecycle management, including creation, LDAP configuration, service deployment, and registration with the proxy infrastructure. -- [`intern-phxdc-pve1/register-container.sh`](intern-phxdc-pve1/register-container.sh): - Proxmox hook script to register new containers with the NGINX proxy and assign HTTP/SSH ports. +- [`ci-cd automation/`](ci-cd%20automation/): + Automation scripts for continuous integration and deployment workflows, including container existence checks, updates, and cleanup operations with helper utilities. -- [`intern-phxdc-pve1/register_proxy_hook.sh`](intern-phxdc-pve1/register_proxy_hook.sh): - Proxmox event hook to trigger container registration on startup. +### Authentication & Directory Services + +- [`LDAP/`](LDAP/): + Contains LDAP authentication infrastructure including a custom Node.js LDAP server that bridges database user management with LDAP protocols, and automated LDAP client configuration tools for seamless container authentication integration. LDAP Server configured to reference the [Proxmox VE Users @pve realm](https://pve.proxmox.com/wiki/User_Management) with optional [Push Notification 2FA](https://github.com/mieweb/mieweb_auth_app) + +### GitHub Action Integration + +- [`proxmox-launchpad/`](proxmox-launchpad/): + The Proxmox LaunchPad GitHub Action for automated container deployment directly from GitHub repositories, supporting both single and multi-component applications. ## How It Works -- **DNS**: All `*.opensource.mieweb.com` requests are routed to the NGINX proxy via Dnsmasq. -- **Reverse Proxy**: NGINX uses a JavaScript module to dynamically resolve the backend IP and port for each subdomain, based on `/etc/nginx/port_map.json`. -- **Container Registration**: When a new container starts, Proxmox runs a hook script that: - - Waits for the container to get a DHCP lease. - - Assigns available HTTP and SSH ports. - - Updates the NGINX port map and reloads NGINX. - - Sets up port forwarding for SSH access. +- **DNS**: All `*.opensource.mieweb.com` requests are routed to the NGINX proxy via Dnsmasq, providing automatic subdomain resolution for containers. +- **Reverse Proxy**: NGINX uses JavaScript modules to dynamically resolve backend IP addresses and ports for each subdomain, based on the container registry in `/etc/nginx/port_map.json`. +- **Container Lifecycle**: When containers start, Proxmox hooks automatically: + - Wait for DHCP lease assignment + - Allocate available HTTP and SSH ports + - Update the NGINX port mapping and reload configuration + - Configure iptables rules for SSH port forwarding +- **GitHub Integration**: The Proxmox LaunchPad action automates the entire process from repository push to live deployment, including dependency installation, service configuration, and application startup. +- **CI/CD Pipeline**: Automated scripts used by [Proxmox LaunchPad](#proxmox-launchpad) to handle container updates, existence checks, and cleanup operations to maintain a clean and efficient hosting environment. +- **LDAP Server**: All LXC Container Authentication is handled by a centralized LDAP server housed in the cluster. Each Container is configured with SSSD, which communicates with the LDAP server to verify/authenitcate user credentials. This approach is more secure than housing credentials locally. + -## Usage +## Proxmox LaunchPad + +The Proxmox LaunchPad is a powerful GitHub Action that automatically creates, manages, and deploys LXC containers on the Proxmox cluster based on your repository's branch activity. It supports: + +- **Automatic Container Creation**: Creates new containers when branches are created or pushed to +- **Multi-Component Deployments**: Supports applications with multiple services (e.g., frontend + backend) +- **Service Integration**: Automatically installs and configures services like MongoDB, Docker, Redis, PostgreSQL, and more +- **Branch-Based Environments**: Each branch gets its own isolated container environment +- **Automatic Cleanup**: Deletes containers when branches are deleted (e.g., after PR merges) + +The action integrates with the existing infrastructure to provide automatic DNS registration, reverse proxy configuration, and port mapping for seamless access to deployed applications. + +## Opensource Cluster Usage + +### For Infrastructure Management 1. **Clone this repository** to your Proxmox host or configuration management system. 2. **Deploy the configuration files** to their respective locations on your infrastructure. 3. **Ensure dependencies**: - - Proxmox VE with container support. - - NGINX with the `ngx_http_js_module`. - - Dnsmasq. - - `jq` for JSON manipulation. -4. **Register new containers** using the provided hook scripts for automatic proxy and DNS integration. + - Proxmox VE with LXC container support + - NGINX with the `ngx_http_js_module` + - Dnsmasq for DNS and DHCP services +4. **Set up LDAP authentication** using the provided LDAP server and client configuration tools. +5. **Configure container templates** and network settings according to your environment. +6. **Register new containers** using the provided hook scripts for automatic proxy and DNS integration. + +### For GitHub Action Deployment + +1. **Add the Proxmox LaunchPad action** to your repository's workflow file. +2. **Configure repository secrets** for Proxmox credentials and optionally a GitHub PAT. +3. **Set up trigger events** for push, create, and delete operations in your workflow. +4. **Configure deployment properties** in your workflow file for automatic application deployment. +5. **Push to your repository** and watch as containers are automatically created and your application is deployed. + +See the [`proxmox-launchpad/README.md`](proxmox-launchpad/README.md) for detailed setup instructions and configuration options. --- -*Current SME: Carter Myers and other contributors to opensource.mieweb.com * +Contributors: Carter Myers, Maxwell Klema, and Anisha Pant \ No newline at end of file diff --git a/ci-cd automation/Proxmox-Launchpad b/ci-cd automation/Proxmox-Launchpad new file mode 160000 index 00000000..038aff5a --- /dev/null +++ b/ci-cd automation/Proxmox-Launchpad @@ -0,0 +1 @@ +Subproject commit 038aff5ad0eacd9f77935ae8819ab59da13fc981 From 1013d8befa71cdfcdf48eaaef502b44fbcfca32c Mon Sep 17 00:00:00 2001 From: maxklema Date: Tue, 5 Aug 2025 16:42:45 -0400 Subject: [PATCH 24/25] Submodule Fix --- .gitmodules | 1 + proxmox-launchpad | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 160000 proxmox-launchpad diff --git a/.gitmodules b/.gitmodules index a38adba7..d7c6479b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,4 @@ url = https://github.com/mieweb/LDAPServer.git [submodule "ci-cd automation/Proxmox-Launchpad"] path = ci-cd automation/Proxmox-Launchpad + url = https://github.com/maxklema/proxmox-launchpad.git diff --git a/proxmox-launchpad b/proxmox-launchpad deleted file mode 160000 index 038aff5a..00000000 --- a/proxmox-launchpad +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 038aff5ad0eacd9f77935ae8819ab59da13fc981 From bc5a8531927734317b21f1a2074d3178ea3b8084 Mon Sep 17 00:00:00 2001 From: maxklema Date: Tue, 5 Aug 2025 18:18:54 -0400 Subject: [PATCH 25/25] Container Ownership Refactoring + misc. fixes --- .../verify_container_ownership.sh | 27 +++++-------------- container creation/get-deployment-details.sh | 4 +-- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/ci-cd automation/helper-scripts/verify_container_ownership.sh b/ci-cd automation/helper-scripts/verify_container_ownership.sh index a0727cc2..f99849ae 100755 --- a/ci-cd automation/helper-scripts/verify_container_ownership.sh +++ b/ci-cd automation/helper-scripts/verify_container_ownership.sh @@ -1,14 +1,9 @@ #!/bin/bash # Script to verify container ownership based on name and CTID -# Last Modified by Maxwell Klema on July 13th, 2025 +# Last Modified by Maxwell Klema on August 5th, 2025 # ----------------------------------------------------- CONTAINER_NAME="${CONTAINER_NAME,,}" - -if [ -z "$CONTAINER_NAME" ]; then - read -p "Enter Container Name → " CONTAINER_NAME -fi - CONTAINER_ID=$( { pct list; ssh root@10.15.0.5 'pct list'; } | awk -v name="$CONTAINER_NAME" '$3 == name {print $1}') if [ -z "$CONTAINER_ID" ]; then @@ -16,22 +11,14 @@ if [ -z "$CONTAINER_ID" ]; then return 1 fi -if [ "$TYPE_RUNNER" != "true" ]; then - if (( $CONTAINER_ID % 2 == 0 )); then - CONTAINER_OWNERSHIP=$(ssh root@10.15.0.5 "pct config \"$CONTAINER_ID\" | grep "tags" | grep \"$PROXMOX_USERNAME\"") - else - CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep -E "(^|;)$PROXMOX_USERNAME(;|$)") - fi -else - CONTAINER_OWNERSHIP=$(ssh root@10.15.0.5 "pct config \"$CONTAINER_ID\" | grep "tags" | grep \"$PROXMOX_USERNAME\"") +CONTAINER_OWNERSHIP=$(ssh root@10.15.20.69 -- "jq '.\"$CONTAINER_NAME\".user' /etc/nginx/port_map.json") +if [ "$TYPE_RUNNER" == "true" ] && (( $CONTAINER_ID % 2 == 0 )); then PVE1="false" - if [ -z "$CONTAINER_OWNERSHIP" ]; then - CONTAINER_OWNERSHIP=$(pct config "$CONTAINER_ID" | grep "tags" | grep -E "(^|;)$PROXMOX_USERNAME(;|$)") - PVE1="true" - fi +elif [ "$TYPE_RUNNER" == "true" ] && (( $CONTAINER_ID % 2 != 0 )); then + PVE1="true" fi -if [ -z "$CONTAINER_OWNERSHIP" ]; then +if [ "$CONTAINER_OWNERSHIP" == "null" ]; then echo "❌ You do not own the container with name \"$CONTAINER_NAME\"." outputError 1 "You do not own the container with name \"$CONTAINER_NAME\"." -fi +fi \ No newline at end of file diff --git a/container creation/get-deployment-details.sh b/container creation/get-deployment-details.sh index 2f5f4215..992cd256 100755 --- a/container creation/get-deployment-details.sh +++ b/container creation/get-deployment-details.sh @@ -51,7 +51,7 @@ if [ -z "$PROJECT_BRANCH" ]; then writeLog "Prompted for project branch" fi -if [ "$PROJECT_BRANCH" == "" ]; then +if [ -z "$PROJECT_BRANCH" ]; then PROJECT_BRANCH="main" writeLog "Using default branch: main" fi @@ -68,7 +68,7 @@ while [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; do echo "⚠️ The branch you provided, \"$PROJECT_BRANCH\", does not exist on repository at \"$PROJECT_REPOSITORY\"." writeLog "Invalid branch entered: $PROJECT_BRANCH" read -p "🪾 Enter the project branch to deploy from (leave blank for \"main\") → " PROJECT_BRANCH - if [ "$PROJECT_BRANCH" == "" ]; then + if [ -z "$PROJECT_BRANCH" ]; then PROJECT_BRANCH="main" fi REPOSITORY_BRANCH_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/$PROJECT_REPOSITORY_SHORTENED/tree/$PROJECT_BRANCH)