diff --git a/.gitmodules b/.gitmodules index 8a790669..4b101069 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "ci-cd-automation/Proxmox-Launchpad"] path = ci-cd-automation/Proxmox-Launchpad url = https://github.com/maxklema/proxmox-launchpad.git +[submodule "mie-opensource-landing"] + path = mie-opensource-landing + url = https://github.com/maxklema/mie-opensource-landing diff --git a/README.md b/README.md index bbbc5a2f..8d451969 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Configuration storage for the [opensource.mieweb.org](https://opensource.mieweb.org:8006) Proxmox project. +To learn everything there is about our cluster, see our documentation at [https://opensource.mieweb.org/docs/intro](https://opensource.mieweb.org/docs/intro). + This repository contains configuration files and scripts for managing a Proxmox-based container hosting environment, including automated DNS, NGINX reverse proxy, dynamic port mapping, and the Proxmox LaunchPad GitHub Action for automated container deployment. ## Cluster Graph @@ -99,6 +101,11 @@ If you have an account in the [opensource-mieweb](https://opensource.mieweb.org: - Use the Command Line: ssh create-container@opensource.mieweb.org (mie123!) - Use the Proxmox LaunchPad Github Action to automatically provision, update, and delete containers for you: [Proxmox LaunchPad](#proxmox-launchpad) +## MIE Opensource Landing + +Contains all the source code for [https://opensource.mieweb.org's](https://opensource.mieweb.org) landing page, built with React + Docusaurus. +- Documentation is located at [https://opensource.mieweb.org/docs/intro](https://opensource.mieweb.org/docs/intro). + ## How It Works - **DNS**: All `*.opensource.mieweb.com` requests are routed to the NGINX proxy via Dnsmasq, providing automatic subdomain resolution for containers. diff --git a/ci-cd-automation/check-container-exists.sh b/ci-cd-automation/check-container-exists.sh index 77c8fd8f..ba0b2c9a 100644 --- a/ci-cd-automation/check-container-exists.sh +++ b/ci-cd-automation/check-container-exists.sh @@ -32,13 +32,13 @@ REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") # Check if repository folder is present. if [ "$PVE1" == "true" ]; then - if pct exec $CONTAINER_ID -- test -f /root/container-updates.log; then + if [ ! -z "$CONTAINER_ID" ] && pct exec $CONTAINER_ID -- test -f /root/container-updates.log; then exit 2; # Update Repository else exit 0; # Clone Repository fi else - if ssh 10.15.0.5 "pct exec $CONTAINER_ID -- test -f /root/container-updates.log"; then + if [ ! -z "$CONTAINER_ID" ] && ssh 10.15.0.5 "pct exec $CONTAINER_ID -- test -f /root/container-updates.log" ; then exit 2; # Update Repository else exit 0; # Clone Repository diff --git a/ci-cd-automation/helper-scripts/PVE_user_authentication.sh b/ci-cd-automation/helper-scripts/PVE_user_authentication.sh index a7b84f34..f2f07204 100755 --- a/ci-cd-automation/helper-scripts/PVE_user_authentication.sh +++ b/ci-cd-automation/helper-scripts/PVE_user_authentication.sh @@ -14,7 +14,7 @@ if [ -z "$PROXMOX_PASSWORD" ]; then echo "" fi -USER_AUTHENTICATED=$(ssh root@10.15.234.122 "node /root/bin/js/runner.js authenticateUser \"$PROXMOX_USERNAME\" \"$PROXMOX_PASSWORD\"") +USER_AUTHENTICATED=$(ssh root@create-container "node /root/bin/js/runner.js authenticateUser \"$PROXMOX_USERNAME\" \"$PROXMOX_PASSWORD\"") if [ $USER_AUTHENTICATED == 'false' ]; then outputError 1 "Your Proxmox account, $PROXMOX_USERNAME@pve, was not authenticated. Retry with valid credentials." diff --git a/container-creation/setup-runner.sh b/ci-cd-automation/setup-runner.sh similarity index 98% rename from container-creation/setup-runner.sh rename to ci-cd-automation/setup-runner.sh index 382f4f87..aaf9fafe 100644 --- a/container-creation/setup-runner.sh +++ b/ci-cd-automation/setup-runner.sh @@ -7,7 +7,7 @@ outputError() { echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" echo -e "${BOLD}${MAGENTA}❌ Script Failed. Exiting... ${RESET}" echo -e "$2" - echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" exit $1 } @@ -26,7 +26,7 @@ source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh #Ensure if [ ! -z "$CONTAINER_OWNERSHIP" ]; then outputError 1 "You already own a container with name \"$CONTAINER_NAME\". Please delete it before creating a new one." fi - + # Cloning Container Template and Setting it up ===== REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") @@ -117,8 +117,6 @@ echo "$PUB_KEY" >> /home/update-container/.ssh/authorized_keys echo "$PUB_KEY" >> /home/delete-container/.ssh/authorized_keys echo "$PUB_KEY" >> /home/container-exists/.ssh/authorized_keys -ssh root@10.15.234.122 "echo \"$PUB_KEY\" >> /root/.ssh/authorized_keys" - echo "🔑 Creating Service File..." pct exec $NEXT_ID -- bash -c "cat < /etc/systemd/system/github-runner.service [Unit] diff --git a/ci-cd-automation/update-container.sh b/ci-cd-automation/update-container.sh index 484f4996..0be5e739 100644 --- a/ci-cd-automation/update-container.sh +++ b/ci-cd-automation/update-container.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to automatically fetch new contents from a branch, push them to container, and restart intern -# Last Modified on August 5th, 2025 by Maxwell Klema +# Last Modified on August 17th, 2025 by Maxwell Klema # ---------------------------------------- RESET="\033[0m" @@ -65,7 +65,7 @@ if [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; then fi -# # Get Project Root Directroy +# Get Project Root Directroy if [ "$PROJECT_ROOT" == "." ] || [ -z "$PROJECT_ROOT" ]; then PROJECT_ROOT="/" @@ -116,7 +116,6 @@ if [ ! -z "$HTTP_PORT" ]; then && mv -f /tmp/port_map.json.new /etc/nginx/port_map.json " fi - # Clone repository if needed ==== if (( "$CONTAINER_ID" % 2 == 0 )); then @@ -137,6 +136,64 @@ fi EOF fi +# Update Environment Variables + +if [ ! -z "$RUNTIME_LANGUAGE" ] && echo "$RUNTIME_LANGUAGE" | jq . >/dev/null 2>&1; then # If RUNTIME_LANGUAGE is set and is valid JSON + MULTI_COMPONENT="Y" +fi + +# Helper Function to write environment variables to a file inside container +writeEnvToFile() { + env_file_path="$1" + component_path="$2" + env_vars=$(cat "$env_file_path") + if (( $CONTAINER_ID % 2 == 0 )); then + ssh 10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'if [ ! -f \"$component_path/.env\" ]; then touch \"$component_path/.env\"; fi; echo \"$env_vars\" >> \"$component_path/.env\"'" + else + pct exec $CONTAINER_ID -- bash -c "if [ ! -f \"$component_path/.env\" ]; then touch \"$component_path/.env\"; fi; echo \"$env_vars\" >> \"$component_path/.env\"" + fi +} + +# Check If there are environment variables +if [ ! -z "$CONTAINER_ENV_VARS" ]; then + # generate random temp .env folder to store all env files for different components + RANDOM_NUM=$(shuf -i 100000-999999 -n 1) + ENV_FOLDER="env_$RANDOM_NUM" + ENV_FOLDER_PATH="/var/lib/vz/snippets/container-env-vars/$ENV_FOLDER" + mkdir -p "$ENV_FOLDER_PATH" + + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then # Multi-Component + if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success + for key in $(echo "$CONTAINER_ENV_VARS" | jq -r 'keys[]'); do + COMPONENT_PATH="/root/$REPO_BASE_NAME/$PROJECT_ROOT/$key" + ENV_FILE_NAME=$(echo "$COMPONENT_PATH" | tr '/' '_') + ENV_FILE_NAME="$ENV_FILE_NAME.txt" + ENV_FILE_PATH="$ENV_FOLDER_PATH/$ENV_FILE_NAME" + touch "$ENV_FILE_PATH" + echo "$CONTAINER_ENV_VARS" | jq -r --arg key "$key" '.[$key] | to_entries[] | "\(.key)=\(.value)"' > "$ENV_FILE_PATH" + writeEnvToFile "$ENV_FILE_PATH" "$COMPONENT_PATH" + done + else + outputError "Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." + writeLog "Invalid JSON in CONTAINER_ENV_VARS (GH_ACTION mode)" + exit 16 + fi + else # Single Component + ENV_FILE="env_$RANDOM_NUM.txt" + ENV_FILE_PATH="$ENV_FOLDER_PATH/$ENV_FILE" + touch "$ENV_FILE_PATH" + if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success + COMPONENT_PATH="/root/$REPO_BASE_NAME/$PROJECT_ROOT" + echo "$CONTAINER_ENV_VARS " | jq -r 'to_entries[] | "\(.key)=\(.value)"' > "$ENV_FILE_PATH" #k=v pairs + writeEnvToFile "$ENV_FILE_PATH" "$COMPONENT_PATH" + else + outputError "Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." + writeLog "Invalid JSON in CONTAINER_ENV_VARS for single component (GH_ACTION mode)" + exit 16 + fi + fi +fi + # Update Container with New Contents from repository ===== startComponentPVE1() { @@ -148,15 +205,15 @@ startComponentPVE1() { INSTALL_CMD="$5" if [ "${RUNTIME^^}" == "NODEJS" ]; then - pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD && $BUILD_CMD" > /dev/null 2>&1 - pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 elif [ "${RUNTIME^^}" == "PYTHON" ]; then - pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $INSTALL_CMD && $BUILD_CMD" > /dev/null 2>&1 - pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 fi } @@ -174,22 +231,17 @@ startComponentPVE2() { pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 - " + " elif [ "${RUNTIME^^}" == "PYTHON" ]; then ssh root@10.15.0.5 " pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 - " + " fi } - -if [ ! -z "$RUNTIME_LANGUAGE" ] && echo "$RUNTIME_LANGUAGE" | jq . >/dev/null 2>&1; then # If RUNTIME_LANGUAGE is set and is valid JSON - MULTI_COMPONENT="Y" -fi - if [ "${MULTI_COMPONENT^^}" == "Y" ]; then for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') @@ -208,17 +260,23 @@ if [ "${MULTI_COMPONENT^^}" == "Y" ]; then done if [ ! -z "$ROOT_START_COMMAND" ]; then if (( $CONTAINER_ID % 2 == 0 )); then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" else - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" fi fi # startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." else if (( $CONTAINER_ID % 2 == 0 )); then startComponentPVE2 "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." "$INSTALL_COMMAND" + if [ ! -z "$ROOT_START_COMMAND" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi else startComponentPVE1 "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." "$INSTALL_COMMAND" + if [ ! -z "$ROOT_START_COMMAND" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi fi fi @@ -262,5 +320,5 @@ QUOTED_CMD=$(printf ' %q' "${CMD[@]}") tmux new-session -d -s "$CONTAINER_NAME" "$QUOTED_CMD" echo "✅ Container $CONTAINER_ID has been updated with new contents from branch \"$PROJECT_BRANCH\" on repository \"$PROJECT_REPOSITORY\"." +echo "Wait a few minutes for all background processes to complete before accessing the container." exit 0 - diff --git a/container-creation/create-container.sh b/container-creation/create-container.sh index 16013e6e..66ac88c2 100644 --- a/container-creation/create-container.sh +++ b/container-creation/create-container.sh @@ -42,7 +42,7 @@ echoContainerDetails() { echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" echo -e "${BOLD}${BLUE}Still not working? Contact Max K. at maxklema@gmail.com${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" - + } trap cleanup SIGINT SIGTERM SIGHUP @@ -71,6 +71,7 @@ SERVICES_BASE_FILE="${16}" LINUX_DISTRO="${17}" MULTI_COMPONENTS="${18}" ROOT_START_COMMAND="${19}" +SELF_HOSTED_RUNNER="${20}" # Pick the correct template to clone ===== @@ -96,7 +97,7 @@ fi # Create the Container Clone ==== -if [ "${GH_ACTION^^}" != "Y" ]; then +if [ "${GH_ACTION^^}" != "Y" ] || [ "${SELF_HOSTED_RUNNER^^}" == "N" ]; then CONTAINER_ID=$(pvesh get /cluster/nextid) #Get the next available LXC ID echo "⏳ Cloning Container..." @@ -178,10 +179,10 @@ pct exec $CONTAINER_ID -- bash -c "cd /root && touch container-updates.log" # Run Contianer Provision Script to add container to port_map.json echo "⏳ Running Container Provision Script..." if [ -f "/var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE" ]; then - /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE "$USERNAME_ONLY" "$ROOT_PSWD" + /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE "$USERNAME_ONLY" rm -rf /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE > /dev/null 2>&1 else - /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT "" "$PROXMOX_USERNAME" "$ROOT_PSWD" + /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT "" "$PROXMOX_USERNAME" fi SSH_PORT=$(iptables -t nat -S PREROUTING | grep "to-destination $CONTAINER_IP:22" | awk -F'--dport ' '{print $2}' | awk '{print $1}' | head -n 1 || true) diff --git a/container-creation/deployOnStart.sh b/container-creation/deployOnStart.sh index 82d01a62..294dbd2c 100755 --- a/container-creation/deployOnStart.sh +++ b/container-creation/deployOnStart.sh @@ -1,24 +1,27 @@ #!/bin/bash # Automation Script for attempting to automatically deploy projects and services on a container -# Last Modifided by Maxwell Klema on July 16th, 2025 +# Last Modifided by Maxwell Klema on August 16th, 2025 # ----------------------------------------------------- echo "🚀 Attempting Automatic Deployment" REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") -# Clone github repository from correct branch ==== +# Helper function to normalize paths by removing duplicate slashes +normalize_path() { + echo "$1" | sed 's#/\+#/#g' +} -echo "Repo base name: $REPO_BASE_NAME" +# Clone github repository from correct branch ==== pct enter $CONTAINER_ID < /dev/null 2>&1 && \ cd /root/$REPO_BASE_NAME && \ -git checkout $PROJECT_BRANCH > /dev/null +git checkout $PROJECT_BRANCH > /dev/null 2>&1 else -cd /root/$REPO_BASE_NAME && git fetch && git pull && \ -git checkout $PROJECT_BRANCH +cd /root/$REPO_BASE_NAME && git fetch > /dev/null 2>&1 && git pull > /dev/null 2>&1 && \ +git checkout $PROJECT_BRANCH > /dev/null 2>&1 fi EOF @@ -28,35 +31,98 @@ pct exec $CONTAINER_ID -- bash -c "chmod 700 ~/.bashrc" # enable full R/W/X perm ENV_BASE_FOLDER="/var/lib/vz/snippets/container-env-vars/${ENV_BASE_FOLDER}" -if [ ! -d "$ENV_BASE_FOLDER"]; then +echo "$REPO_BASE_NAME" +echo "$PROJECT_ROOT" +if [ -d "$ENV_BASE_FOLDER" ]; then if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then for FILE in $ENV_BASE_FOLDER/*; do FILE_BASENAME=$(basename "$FILE") FILE_NAME="${FILE_BASENAME%.*}" ENV_ROUTE=$(echo "$FILE_NAME" | tr '_' '/') # acts as the route to the correct folder to place .env file in. - + ENV_VARS=$(cat $ENV_BASE_FOLDER/$FILE_BASENAME) - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$ENV_ROUTE && echo "$ENV_VARS" > .env" > /dev/null 2>&1 + COMPONENT_PATH=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT/$ENV_ROUTE") + pct exec $CONTAINER_ID -- bash -c "if [ ! -f \"$COMPONENT_PATH/.env\" ]; then touch \"$COMPONENT_PATH/.env\"; fi; echo \"$ENV_VARS\" >> \"$COMPONENT_PATH/.env\"" > /dev/null 2>&1 done else ENV_FOLDER_BASE_NAME=$(basename "$ENV_BASE_FOLDER") ENV_VARS=$(cat $ENV_BASE_FOLDER/$ENV_FOLDER_BASE_NAME.txt || true) - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && echo "$ENV_VARS" > .env" > /dev/null 2>&1 + COMPONENT_PATH=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") + pct exec $CONTAINER_ID -- bash -c "if [ ! -f \"$COMPONENT_PATH/.env\" ]; then touch \"$COMPONENT_PATH/.env\"; fi; echo \"$ENV_VARS\" >> \"$COMPONENT_PATH/.env\"" > /dev/null 2>&1 fi fi +# Install Specific Runtime Versions (if Needed) + +echo "VERSIONS_DICT: $VERSIONS_DICT" +echo "RUNTIME_LANGUAGE: $RUNTIME_LANGUAGE" + +pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 8 > /dev/null 2>&1 + +# Function to handle runtime installation +install_runtime() { + local runtime_language=$1 + local version=$2 + + if [ "${runtime_language,,}" == "nodejs" ]; then + local major=$(echo "$version" | cut -d. -f1) + local node_version_exists=$(curl -s https://nodejs.org/dist/index.json | grep "version\":\"v$major") + if [ ! -z "$node_version_exists" ]; then + source /var/lib/vz/snippets/helper-scripts/node_runtime_install.sh "$major" + else + echo "Node.js version $version ($major) is not available. Please check the version number. Using latest version instead." + fi + elif [ "${runtime_language,,}" == "python" ]; then + IFS='.' read -r -a parts <<< "$version" + + # Fill missing parts with 0 + while [ "${#parts[@]}" -lt 3 ]; do + parts+=("0") + done + + version="${parts[0]}.${parts[1]}.${parts[2]}" + local python_version_exists=$(curl -s https://www.python.org/ftp/python/ | grep "$version") + if [ ! -z "$python_version_exists" ]; then + source /var/lib/vz/snippets/helper-scripts/python_runtime_install.sh "${LINUX_DISTRO,,}" "$version" + else + echo "Python version $version is not available. Please check the version number. Using latest version instead." + fi + fi +} + +for key in $(echo "$VERSIONS_DICT" | jq -r 'keys[]'); do + if [ "$key" == "default" ] && [ "${MULTI_COMPONENT^^}" != "Y" ]; then + version=$(echo "$VERSIONS_DICT" | jq --arg k "$key" '.[$k]') + if [ "$version" != "null" ]; then + version=$(echo "$version" | sed 's/"//g') + install_runtime "$RUNTIME_LANGUAGE" "$version" + fi + else + value=$(echo "$RUNTIME_LANGUAGE" | jq --arg k "$key" '.[$k]') + value=$(echo "$value" | sed 's/"//g') + version=$(echo "$VERSIONS_DICT" | jq --arg k "$key" '.[$k]') + if [ "$version" != "null" ]; then + version=$(echo "$version" | sed 's/"//g') + install_runtime "$value" "$version" + fi + fi +done + # Run Installation Commands ==== runInstallCommands() { RUNTIME="$1" COMP_DIR="$2" + + # Create normalized path + WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR") if [ "${RUNTIME^^}" == "NODEJS" ]; then - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && sudo $INSTALL_CMD" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "cd $WORK_DIR && sudo $INSTALL_CMD" > /dev/null 2>&1 elif [ "${RUNTIME^^}" == "PYTHON" ]; then pct enter $CONTAINER_ID < /dev/null -cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && \ +cd $WORK_DIR && \ python3 -m venv venv && source venv/bin/activate && \ pip install --upgrade pip && \ $INSTALL_CMD @@ -83,4 +149,4 @@ if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then while read line; do pct exec $CONTAINER_ID -- bash -c "$line" > /dev/null 2>&1 done < "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" -fi \ No newline at end of file +fi diff --git a/container-creation/node_runtime_install.sh b/container-creation/node_runtime_install.sh new file mode 100644 index 00000000..21095418 --- /dev/null +++ b/container-creation/node_runtime_install.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Script to install a specific Node.js runtime version +# Last Modified by Maxwell Klema on August 18th, 2025 +# ---------------------------------------------------------- + +pct enter "$CONTAINER_ID" -- < " + echo "Example: $0 debian 3.8.9" + exit 1 +fi + +OS=$1 +PYTHON_VERSION=$2 +PYTHON_URL="https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz" + +# Function to install dependencies based on OS +install_dependencies() { + case "${OS,,}" in + debian) + pct exec $CONTAINER_ID -- bash -c "sudo apt update -y && sudo apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev wget" > /dev/null 2>&1 + ;; + rocky) + pct exec $CONTAINER_ID -- bash -c "sudo dnf update -y && sudo dnf install -y gcc zlib-devel ncurses-devel gdbm-devel nss-devel openssl-devel readline-devel wget libffi-devel" > /dev/null 2>&1 + ;; + esac +} + +# Function to install Python +install_python() { + echo "⏳ Installing Python $PYTHON_VERSION... This may take a while." + pct exec "$CONTAINER_ID" -- bash -c "cd /usr/src && sudo wget $PYTHON_URL && sudo tar xzf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && sudo ./configure --enable-optimizations && \ + sudo make -j$(nproc) && sudo make altinstall && \ + sudo ln -sfn /usr/local/bin/python${PYTHON_VERSION%.*} /usr/local/bin/python3 && \ + python3 -m ensurepip --upgrade && python3 -m pip install --upgrade pip" + echo "Python $PYTHON_VERSION installed successfully." > /dev/null 2>&1 +} + +install_dependencies +install_python diff --git a/container-creation/start_services.sh b/container-creation/start_services.sh index 7942ad02..10f9d651 100644 --- a/container-creation/start_services.sh +++ b/container-creation/start_services.sh @@ -22,6 +22,11 @@ PROJECT_BRANCH="${15}" UPDATE_CONTAINER="${16}" CONTAINER_NAME="${CONTAINER_NAME,,}" +# Helper function to normalize paths by removing duplicate slashes +normalize_path() { + echo "$1" | sed 's#/\+#/#g' +} + if [ "${GH_ACTION^^}" == "Y" ]; then sleep 8 # Wait for Job to Complete fi @@ -48,45 +53,53 @@ if (( $CONTAINER_ID % 2 == 0 )); then if [ "${GH_ACTION^^}" == "Y" ]; then ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- systemctl start github-runner" - fi + fi startProject() { - RUNTIME="$1" - BUILD_CMD="$2" - START_CMD="$3" - COMP_DIR="$4" + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" + + # Create normalized path + WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR") - if [ -z "$BUILD_CMD" ]; then - BUILD_CMD="true" - fi + if [ -z "$BUILD_CMD" ]; then + BUILD_CMD="true" + fi - if [ "${RUNTIME^^}" == "NODEJS" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c \"mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp tmux new-session -d 'export HOME=/root export PATH=\\\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 - elif [ "${RUNTIME^^}" == "PYTHON" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c \"mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp tmux new-session -d 'export HOME=/root export PATH=\\\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 - fi + if [ "${RUNTIME^^}" == "NODEJS" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c \"mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp tmux new-session -d 'export HOME=/root export PATH=\\\$PATH:/usr/local/bin && cd $WORK_DIR && $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c \"mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp tmux new-session -d 'export HOME=/root export PATH=\\\$PATH:/usr/local/bin && cd $WORK_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 + fi - } + } if [ "${DEPLOY_ON_START^^}" == "Y" ]; then - if [ "${MULTI_COMPONENT^^}" == "Y" ]; then - for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do - START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') - BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - if [ "$BUILD" == "null" ]; then - BUILD="" - fi - startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" - done - if [ ! -z "$ROOT_START_COMMAND" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 - fi - else - startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." - fi - fi + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" + done + if [ ! -z "$ROOT_START_COMMAND" ]; then + ROOT_WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd $ROOT_WORK_DIR && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi + else + startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + if [ ! -z "$ROOT_START_COMMAND" ]; then + ROOT_WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd $ROOT_WORK_DIR && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi + fi + fi # PVE 1 else @@ -94,42 +107,50 @@ else sleep 5 if [ "${GH_ACTION^^}" == "Y" ]; then pct exec $CONTAINER_ID -- bash -c "systemctl start github-runner" - fi + fi startComponent() { - RUNTIME="$1" - BUILD_CMD="$2" - START_CMD="$3" - COMP_DIR="$4" - - if [ -z "$BUILD_CMD" ]; then - BUILD_CMD="true" - fi - - if [ "${RUNTIME^^}" == "NODEJS" ]; then - pct exec "$CONTAINER_ID" -- bash -c "mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp/tmux-0 tmux new-session -d \"export HOME=/root && export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $BUILD_CMD && $START_CMD\"" - elif [ "${RUNTIME^^}" == "PYTHON" ]; then - pct exec "$CONTAINER_ID" -- bash -c "mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp/tmux-0 tmux new-session -d \"export HOME=/root &&export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $BUILD_CMD && $START_CMD\"" - fi - } + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" + + # Create normalized path + WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR") + + if [ -z "$BUILD_CMD" ]; then + BUILD_CMD="true" + fi + + if [ "${RUNTIME^^}" == "NODEJS" ]; then + pct exec "$CONTAINER_ID" -- bash -c "mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp/tmux-0 tmux new-session -d \"export HOME=/root && export PATH=\$PATH:/usr/local/bin && cd $WORK_DIR && $BUILD_CMD && $START_CMD\"" + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + pct exec "$CONTAINER_ID" -- bash -c "mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp/tmux-0 tmux new-session -d \"export HOME=/root &&export PATH=\$PATH:/usr/local/bin && cd $WORK_DIR && source venv/bin/activate && $BUILD_CMD && $START_CMD\"" + fi + } pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 > /dev/null #temporarily bump up container resources for computation hungry processes (e.g. meteor) - if [ "${MULTI_COMPONENT^^}" == "Y" ]; then - for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do - START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') - BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - if [ "$BUILD" == "null" ]; then - BUILD="" - fi - - startComponent "$RUNTIME" "$BUILD" "$START" "$COMPONENT" - done - if [ ! -z "$ROOT_START_COMMAND" ]; then - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" > /dev/null 2>&1 - fi - else - startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." - fi + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + + startComponent "$RUNTIME" "$BUILD" "$START" "$COMPONENT" + done + if [ ! -z "$ROOT_START_COMMAND" ]; then + ROOT_WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") + pct exec $CONTAINER_ID -- bash -c "cd $ROOT_WORK_DIR && $ROOT_START_COMMAND" > /dev/null 2>&1 + fi + else + startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + if [ ! -z "$ROOT_START_COMMAND" ]; then + ROOT_WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") + pct exec $CONTAINER_ID -- bash -c "cd $ROOT_WORK_DIR && $ROOT_START_COMMAND" > /dev/null 2>&1 + fi + fi fi diff --git a/mie-opensource-landing b/mie-opensource-landing new file mode 160000 index 00000000..31c40c17 --- /dev/null +++ b/mie-opensource-landing @@ -0,0 +1 @@ +Subproject commit 31c40c1717789132308c175a6961688edef221dd diff --git a/nginx-reverse-proxy/port-map-server.js b/nginx-reverse-proxy/port-map-server.js new file mode 100644 index 00000000..a935eb27 --- /dev/null +++ b/nginx-reverse-proxy/port-map-server.js @@ -0,0 +1,67 @@ +// etc/nginx/port-map-server.js +// JSON-server that returns all (filtered) hosts to the https://opensource.mieweb.org +// Last modified on Aug 22, 2025 by Maxwell Klema + +const jsonServer = require('json-server'); +const path = require('path'); +const fs = require('fs'); +const filePath = "/etc/nginx/port_map.json"; + +const server = jsonServer.create(); +const router = jsonServer.router(path.join(__dirname, 'port_map.json')); +const middlewares = jsonServer.defaults(); +server.use(middlewares); + +server.get('/keys', (req, res) => { + const db = router.db; + const keys = Object.keys(db.getState()).filter(key => !['container-creation', 'intern-dnsmasq', 'wazuh-server', 'wazuh-indexer', 'wazuh-dashboard', 'intern-nginx', 'mie-ldap-server', 'create-a-container', 'landing-page'].includes(key)); + + // Filter out keys that are branches of a main/master for a repository + + let content = fs.readFileSync(filePath); + let cachedMapping = JSON.parse(content); + + all_hosts = []; + proxmox_launchpad_lxcs = []; + exclude = []; + for (const key of Object.keys(cachedMapping)) { + all_hosts.push(key); + if (key.endsWith("-main") || key.endsWith("master")) { + proxmox_launchpad_lxcs.push(key.substring(0, key.lastIndexOf("-"))); + } + } + + for (const entry of proxmox_launchpad_lxcs) { + hosts_to_filter = all_hosts.filter(key => { + return key.startsWith(entry) && (!key.endsWith("-main") || !key.startsWith(entry)); + }); + hosts_to_filter.forEach(host => exclude.push(host)); + } + + const filteredKeys = keys.filter(key => !exclude.includes(key)); + res.json(filteredKeys); +}) + +server.get('/:key', (req, res) => { + const key = req.params.key; + const db = router.db; + const value = db.getState()[key]; + + const response = { + name: key, + owner: value.user, + description: value.description || "", + github_url: value.github_url || "", + }; + + if (value) { + res.json(response); + } else { + res.status(404).json({ error: 'Not found' }); + } +}); + +server.use(router); +server.listen(3001, () => { + console.log("JSON Server Running on http://localhost:3001"); +})