feat: Add support for installing Python extra packages

.github/ISSUE_TEMPLATE/bug_report.md

@@ -28,7 +28,7 @@ The full log with the outputted error.
 
 - Host OS: [e.g. `uname -a`]
 - Docker: [e.g. `docker --version`]
-- Image tag: [e.g. `3007.0`]
+- Image tag: [e.g. `3007.0_1`]
 
 **Additional context**
 Add any other context about the problem here.

.github/workflows/build-and-test.yml

@@ -142,6 +142,11 @@ jobs:
           sudo systemctl disable salt-minion
           sudo rm -f /var/log/salt/minion
 
+      - name: Install tests utils
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y jq
+
       - name: Execute basic tests
         if: always()
         run: tests/basic/test.sh
@@ -180,6 +185,10 @@ jobs:
         if: always()
         run: tests/gpg/test.sh
 
+      - name: Python Extra Packages tests
+        if: always()
+        run: tests/python-extra-packages/test.sh
+
       - name: Cleanup
         if: always()
         run: |

CHANGELOG.md

@@ -4,6 +4,10 @@ This file only reflects the changes that are made in this image.
 Please refer to the [Salt 3007.0 Release Notes](https://docs.saltstack.com/en/latest/topics/releases/3007.0.html)
 for the list of changes in SaltStack.
 
+**3007.0_1**
+
+- Add support for installing Python extra packages ([#234](https://github.com/cdalvaro/docker-salt-master/issues/234) for more details).
+
 **3007.0**
 
 - Upgrade `salt-master` to `3007.0` *Chlorine*.

Dockerfile

@@ -5,7 +5,7 @@ ARG VCS_REF
 
 # https://github.com/saltstack/salt/releases
 ENV SALT_VERSION="3007.0"
-ENV IMAGE_REVISION=
+ENV IMAGE_REVISION="_1"
 ENV IMAGE_VERSION="${SALT_VERSION}${IMAGE_REVISION}"
 
 ENV SALT_DOCKER_DIR="/etc/docker-salt" \

README.md

@@ -28,7 +28,7 @@ Automated builds of the image are available on
 the recommended method of installation.
 
 ```sh
-docker pull ghcr.io/cdalvaro/docker-salt-master:3007.0
+docker pull ghcr.io/cdalvaro/docker-salt-master:3007.0_1
 ```
 
 You can also pull the `latest` tag, which is built from the repository `HEAD`
@@ -573,6 +573,40 @@ docker exec -it salt_stack /sbin/entrypoint.sh app:reload-3rd-formulas
 `file_roots` base configuration file will be updated with current existing formulas and `salt-master` service will be
 restarted to reload the new configuration.
 
+### Python Extra Packages
+
+Some formulas may depend on Python packages that are not included in the default Salt installation. You can add these packages by setting the `PYTHON_PACKAGES_FILE` environment variable with an absolute path pointing to a `requirements.txt` file inside the container.
+
+```sh
+docker run --name salt_master --detach \
+    --publish 4505:4505 --publish 4506:4506 \
+    --env SALT_LOG_LEVEL="info" \
+    --env PYTHON_PACKAGES_FILE=/home/salt/data/other/requirements.txt \
+    --volume $(pwd)/roots/:/home/salt/data/srv/ \
+    --volume $(pwd)/keys/:/home/salt/data/keys/ \
+    --volume $(pwd)/logs/:/home/salt/data/logs/ \
+    --volume $(pwd)/requirements.txt:/home/salt/data/other/requirements.txt \
+    ghcr.io/cdalvaro/docker-salt-master:latest
+```
+
+This will install the packages listed in the `requirements.txt` file into the container
+before `salt-master` starts.
+
+Alternatively, you can set the `PYTHON_PACKAGES` environment variable with a list of Python packages to be installed.
+
+```sh
+docker run --name salt_master --detach \
+    --publish 4505:4505 --publish 4506:4506 \
+    --env SALT_LOG_LEVEL="info" \
+    --env PYTHON_PACKAGES="docker==7.0.0 redis" \
+    --volume $(pwd)/roots/:/home/salt/data/srv/ \
+    --volume $(pwd)/keys/:/home/salt/data/keys/ \
+    --volume $(pwd)/logs/:/home/salt/data/logs/ \
+    ghcr.io/cdalvaro/docker-salt-master:latest
+```
+
+Although both methods are supported, they are mutually exclusive. If both are set, `PYTHON_PACKAGES_FILE` will take precedence.
+
 ### Logs
 
 `salt-master` output is streamed directly to the container's `stdout` and `stderr`. However, they are also written inside `/home/salt/data/logs/`.
@@ -691,6 +725,8 @@ installation.
 | `TIMEZONE` / `TZ`                                                                                                                     | Set the container timezone. Defaults to `UTC`. Values are expected to be in Canonical format. Example: `Europe/Madrid`. See the list of [acceptable values](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones).                                  |
 | `PUID`                                                                                                                                | Sets the uid for user `salt` to the specified uid. Default: `1000`.                                                                                                                                                                                         |
 | `PGID`                                                                                                                                | Sets the gid for user `salt` to the specified gid. Default: `1000`.                                                                                                                                                                                         |
+| `PYTHON_PACKAGES`                                                                                                                     | Contains a list of Python packages to be installed. Default: _Unset_.                                                                                                                                                                                       |
+| `PYTHON_PACKAGES_FILE`                                                                                                                | An absolute path inside the container pointing to a requirements.txt file for installing Python extra packages. Takes preference over: `PYTHON_PACKAGES`. Default: _Unset_                                                                                  |
 | `SALT_RESTART_MASTER_ON_CONFIG_CHANGE`                                                                                                | Set this to `True` to restart `salt-master` service when configuration files change. Default: `False`.                                                                                                                                                      |
 | [`SALT_LOG_LEVEL`](https://docs.saltproject.io/en/latest/ref/configuration/master.html#log-level)                                     | The level of messages to send to the console. One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'. Default: `warning`.                                                                                                                |
 | `SALT_LOG_ROTATE_FREQUENCY`                                                                                                           | Logrotate frequency for salt logs. Available options are 'daily', 'weekly', 'monthly', and 'yearly'. Default: `weekly`.                                                                                                                                     |

assets/runtime/functions.sh

@@ -10,6 +10,7 @@ source "${ENV_DEFAULTS_FILE}"
 # cdalvaro managed block string
 SELF_MANAGED_BLOCK_STRING="## cdalvaro managed block"
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  exec_as_salt
 #   DESCRIPTION:  Execute the pass command as the `SALT_USER` user.
@@ -23,6 +24,7 @@ function exec_as_salt()
   fi
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  log_debug
 #   DESCRIPTION:  Echo debug information to stdout.
@@ -33,6 +35,7 @@ function log_debug() {
   fi
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  log_info
 #   DESCRIPTION:  Echo information to stdout.
@@ -41,6 +44,7 @@ function log_info() {
   echo "[INFO] - $*"
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  log_warn
 #   DESCRIPTION:  Echo warning information to stdout.
@@ -49,6 +53,7 @@ function log_warn() {
   (>&2 echo "[WARN] - $*")
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  log_error
 #   DESCRIPTION:  Echo errors to stderr.
@@ -58,6 +63,7 @@ function log_error()
   (>&2 echo "[ERROR] - $*")
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  map_uidgid
 #   DESCRIPTION:  Map salt user with host user.
@@ -84,6 +90,7 @@ function map_uidgid()
   fi
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  update_template
 #   DESCRIPTION:  Replace placeholders with values.
@@ -117,6 +124,7 @@ function update_template()
   rm -f "${tmp_file}"
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  configure_timezone
 #   DESCRIPTION:  Configure containers timezone.
@@ -140,6 +148,7 @@ function configure_timezone()
   fi
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  gen_signed_keys
 #   DESCRIPTION:  Generate a master_sign key pair and its signature.
@@ -181,6 +190,7 @@ function gen_signed_keys()
   echo -n "${output_dir}"
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  _setup_master_keys
 #   DESCRIPTION:  Setup salt-master keys.
@@ -223,6 +233,7 @@ function _setup_master_keys()
   fi
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  _setup_master_sign_keys
 #   DESCRIPTION:  Setup salt-master sign keys.
@@ -279,6 +290,7 @@ function _setup_master_sign_keys()
   fi
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  _check_and_link_gpgkey
 #   DESCRIPTION:  Check and link a gpgkey if env variable is set.
@@ -309,6 +321,7 @@ function _check_and_link_gpgkey() {
   ln -sfn "${SOURCE_GPGKEY}" "${TARGET_GPGKEY}"
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  _setup_gpgkeys
 #   DESCRIPTION:  Setup GPG keys.
@@ -353,6 +366,7 @@ function _setup_gpgkeys()
   (echo trust & echo 5 & echo y & echo quit) | exec_as_salt gpg "${GPG_COMMON_OPTS[@]}" --command-fd 0 --edit-key "${key_id}"
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  setup_salt_keys
 #   DESCRIPTION:  Repair keys permissions and creates keys if needed.
@@ -378,6 +392,7 @@ function setup_salt_keys()
   find "${SALT_HOME}" -path "${SALT_KEYS_DIR}/*" -print0 | xargs -0 chown -h "${SALT_USER}":
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  configure_salt_master
 #   DESCRIPTION:  Configure master service.
@@ -410,6 +425,7 @@ function configure_salt_master()
     SALT_MASTER_USE_PUBKEY_SIGNATURE
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  configure_salt_api
 #   DESCRIPTION:  Configure salt-api if service is set to be enabled.
@@ -489,6 +505,7 @@ EOF
 
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  configure_salt_formulas
 #   DESCRIPTION:  Configure salt-formulas.
@@ -512,6 +529,7 @@ function configure_salt_formulas()
   rm "${tmp_file}"
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  initialize_datadir
 #   DESCRIPTION:  Initialize main directories.
@@ -577,6 +595,7 @@ function initialize_datadir()
   chown -R "${SALT_USER}": "${SALT_LOGS_DIR}/salt"
 }
 
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  configure_logrotate
 #   DESCRIPTION:  Configure logrotate.
@@ -642,6 +661,11 @@ EOF
 
 }
 
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  configure_config_reloader
+#   DESCRIPTION:  Configure config reloader.
+#----------------------------------------------------------------------------------------------------------------------
 function configure_config_reloader()
 {
   rm -f /etc/supervisor/conf.d/config-reloader.conf
@@ -663,6 +687,36 @@ stderr_logfile=${SALT_LOGS_DIR}/supervisor/%(program_name)s.log
 EOF
 }
 
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  install_python_additional_packages
+#   DESCRIPTION:  Install additional python packages.
+#----------------------------------------------------------------------------------------------------------------------
+function install_python_additional_packages()
+{
+  if [[ -n "${PYTHON_PACKAGES_FILE}" ]]; then
+    log_info "Installing additional python packages from: ${PYTHON_PACKAGES_FILE} ..."
+
+    if [[ ! -f "${PYTHON_PACKAGES_FILE}" ]]; then
+      log_error "PYTHON_PACKAGES_FILE is set to '${PYTHON_PACKAGES_FILE}' but it doesn't exist."
+      return 1
+    fi
+
+    salt-pip install --no-cache-dir -r "${PYTHON_PACKAGES_FILE}"
+    local RETURN_CODE=$?
+    [[ -z "${PYTHON_PACKAGES}" ]] || log_warn "PYTHON_PACKAGES is set, but it will be ignored because PYTHON_PACKAGES_FILE is set."
+    return "${RETURN_CODE}"
+  fi
+
+  if [[ -n "${PYTHON_PACKAGES}" ]]; then
+    IFS=" " read -ra PYTHON_PACKAGES <<< "${PYTHON_PACKAGES}"
+    log_info "Installing additional python packages: ${PYTHON_PACKAGES[@]} ..."
+    salt-pip install --no-cache-dir "${PYTHON_PACKAGES[@]}"
+    return $?
+  fi
+}
+
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  initialize_system
 #   DESCRIPTION:  Initialize the system.
@@ -678,5 +732,6 @@ function initialize_system()
   configure_salt_formulas
   configure_config_reloader
   setup_salt_keys
+  install_python_additional_packages
   rm -rf /var/run/supervisor.sock
 }