Add mariner2.0 changes in a single commit

common/clear_history.sh

@@ -12,6 +12,10 @@ find_distro() {
     then
         local ubuntu_distro=`find_ubuntu_distro`
         echo "${os} ${ubuntu_distro}"
+    elif [[ $os == "Common Base Linux Mariner" ]]
+    then
+        local mariner_distro=`find_mariner_distro`
+        echo "Mariner ${mariner_distro}"
     else
         echo "*** Error - invalid distro!"
         exit -1
@@ -28,10 +32,15 @@ find_ubuntu_distro() {
     echo `cat /etc/os-release | awk 'match($0, /^PRETTY_NAME="(.*)"/, result) { print result[1] }' | awk '{print $2}' | cut -d. -f1,2`
 }
 
+# Find Mariner distro
+find_mariner_distro() {
+    echo $(cat /etc/os-release | awk 'match($0, /^VERSION_ID="(.*)"/, result) { print result[1] }')
+}
+
 distro=`find_distro`
 echo "Detected distro: ${distro}"
 
-if [[ $distro == *"AlmaLinux"* ]]
+if [[ $distro == *"AlmaLinux"* ]] || [[ $distro == *"Mariner"* ]]
 then
     # Sync yum and rpmdb after installing rpm's outside yum
     yum history sync

common/setup_sku_customizations.sh

@@ -7,14 +7,29 @@ cp $COMMON_DIR/../customizations/* /opt/azurehpc/customizations
 
 ## Copy topology files to /opt/microsoft
 mkdir -p /opt/microsoft
+chmod 644 $COMMON_DIR/../topology/* # Ensure read permission is set of others
 cp $COMMON_DIR/../topology/* /opt/microsoft
 
 ## Systemd service for setting up appropriate customizations based on SKU
 cat <<EOF >/usr/sbin/setup_sku_customizations.sh
 #!/bin/bash
 
 metadata_endpoint="http://169.254.169.254/metadata/instance?api-version=2019-06-04"
-vmSize=\$(curl -H Metadata:true \$metadata_endpoint | jq -r ".compute.vmSize")
+vmSize=\$(curl -s -H Metadata:true \$metadata_endpoint | jq -r ".compute.vmSize")
+
+retry_count=0
+while [ -z "\${vmSize}" ] && (( retry_count++ < 5 ))
+do
+    sleep 30
+    vmSize=\$(curl -s -H Metadata:true \$metadata_endpoint | jq -r ".compute.vmSize")
+done
+
+if [ -z "\${vmSize}" ]
+then
+    echo "Error! Could not retrieve VM Size from IMDS endpoint"
+    exit 1
+fi
+
 vmSize=\$(echo "\$vmSize" | awk '{print tolower(\$0)}')
 
 ## Topo file setup based on SKU
@@ -63,6 +78,13 @@ then
     rmmod nvidia_peermem
 fi
 
+# Mariner only
+if lsmod | grep nv_peer_mem &> /dev/null
+then 
+    rmmod nv_peer_mem
+    rpm -e nvidia_peer_memory
+fi
+
 # Clear topo and graph files
 rm -rf /opt/microsoft/ncv4
 rm -rf /opt/microsoft/ndv2

customizations/ndv4.sh

@@ -26,7 +26,13 @@ then
 fi
 
 ## load nvidia-peermem module
-modprobe nvidia-peermem
+os=$(cat /etc/os-release | grep "^ID=" | awk -F"=" '{print $NF}')
+if [ ${os} == "mariner" ]
+then
+    modprobe nv_peer_mem
+else
+    modprobe nvidia-peermem
+fi
 
 # ## Setup NVME devices
 # if [ ! -f /etc/systemd/system/nvme-raid.service ]; then

customizations/ndv5.sh

@@ -26,4 +26,10 @@ then
 fi
 
 ## load nvidia-peermem module
-modprobe nvidia-peermem
+os=$(cat /etc/os-release | grep "^ID=" | awk -F"=" '{print $NF}')
+if [ ${os} == "mariner" ]
+then
+    modprobe nv_peer_mem
+else
+    modprobe nvidia-peermem
+fi

mariner/common/add-udev-rules.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+#cat << EOF >> /etc/udev/rules.d/60-ib.rules
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file
+#
+# Rename modes:
+# NAME_FALLBACK - Try to name devices in the following order:
+#                 by-pci -> by-guid -> kernel
+# NAME_KERNEL - leave name as kernel provided
+# NAME_PCI - based on PCI/slot/function location
+# NAME_GUID - based on system image GUID
+#
+# The stable names are combination of device type technology and rename mode.
+# Infiniband - ib*
+# RoCE - roce*
+# iWARP - iw*
+# OPA - opa*
+# Default (unknown protocol) - rdma*
+#
+# Example:
+# * NAME_PCI
+#   pci = 0000:00:0c.4
+#   Device type = IB
+#   mlx5_0 -> ibp0s12f4
+# * NAME_GUID
+#   GUID = 5254:00c0:fe12:3455
+#   Device type = RoCE
+#   mlx5_0 -> rocex525400c0fe123455
+#
+#ACTION=="add", SUBSYSTEM=="infiniband", PROGRAM="rdma_rename %k NAME_PCI"
+#EOF

mariner/common/disable_cloudinit.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+
+# Disable Cloud-Init
+cat << EOF >> /etc/cloud/cloud.cfg.d/99-custom-networking.cfg
+network: {config: disabled}
+EOF
+
+# Remove Hardware Mac Address and DHCP Name
+# cp /etc/sysconfig/network-scripts/ifcfg-eth0 tempFile
+# grep -v -E "HWADDR=|DHCP_HOSTNAME=" /etc/sysconfig/network-scripts/ifcfg-eth0 > tempFile
+# mv tempFile /etc/sysconfig/network-scripts/ifcfg-eth0

mariner/common/hpc-tuning.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+set -ex
+
+# Update memory limits
+cat << EOF >> /etc/security/limits.conf
+*               hard    memlock         unlimited
+*               soft    memlock         unlimited
+*               hard    nofile          65535
+*               soft    nofile          65535
+*               hard    stack           unlimited
+*               soft    stack           unlimited
+EOF
+
+# Enable reclaim mode
+echo "vm.zone_reclaim_mode = 1" >> /etc/sysctl.conf
+echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf
+echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf
+echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf
+echo "sunrpc.tcp_max_slot_table_entries = 128" >> /etc/sysctl.conf
+
+## Systemd service for starting sunrpc and adding setting parameters
+cat <<EOF >/usr/sbin/sunrpc_tcp_settings.sh
+#!/bin/bash
+
+modprobe sunrpc
+sysctl -p
+EOF
+
+chmod 755 /usr/sbin/sunrpc_tcp_settings.sh
+
+cat <<EOF >/etc/systemd/system/sunrpc_tcp_settings.service
+[Unit]
+Description=Set sunrpc tcp settings
+
+[Service]
+User=root
+Type=oneshot
+ExecStart=/usr/sbin/sunrpc_tcp_settings.sh
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl enable sunrpc_tcp_settings
+systemctl start sunrpc_tcp_settings
+systemctl is-active --quiet sunrpc_tcp_settings
+
+error_code=$?
+if [ ${error_code} -ne 0 ]
+then
+    echo "sunrpc_tcp_settings service inactive!"
+    exit ${error_code}
+fi
+
+# Remove auoms if exists - Prevent CPU utilization by auoms
+if tdnf list installed azsec-monitor >/dev/null 2>&1; then tdnf remove -y azsec-monitor; fi
+
+# Update WALinuxAgent - for IPoIB
+tdnf update -y WALinuxAgent
+
+# Configure WALinuxAgent
+sed -i -e 's/# OS.EnableRDMA=y/OS.EnableRDMA=y/g' /etc/waagent.conf
+
+function update_waagent_conf {
+    key=$1
+    value=$2
+
+    # Check if the key exists in the file
+    if grep -q "^$key=" /etc/waagent.conf; then
+        # Update the value if the key exists
+        sed -i "s/^$key=.*/$key=$value/" /etc/waagent.conf
+    else
+        # Add the key-value pair if the key does not exist
+        echo "$key=$value" >> /etc/waagent.conf
+    fi
+}
+
+update_waagent_conf "Extensions.GoalStatePeriod" "300"
+update_waagent_conf "Extensions.InitialGoalStatePeriod" "6"
+update_waagent_conf "OS.EnableFirewallPeriod" "300"
+update_waagent_conf "OS.RemovePersistentNetRulesPeriod" "300"
+update_waagent_conf "OS.RootDeviceScsiTimeoutPeriod" "300"
+update_waagent_conf "OS.MonitorDhcpClientRestartPeriod" "60"
+update_waagent_conf "Provisioning.MonitorHostNamePeriod" "60"
+
+systemctl restart waagent
+$COMMON_DIR/write_component_version.sh "waagent" $(waagent --version | head -n 1 | awk -F' ' '{print $1}' | awk -F- '{print $2}')
+
+# Setting Linux NFS read-ahead limits
+# Reference: 
+#    https://learn.microsoft.com/en-us/azure/azure-netapp-files/performance-linux-nfs-read-ahead
+#    https://learn.microsoft.com/en-us/azure/storage/blobs/secure-file-transfer-protocol-support-how-to?tabs=azure-portal
+cat > /etc/udev/rules.d/90-nfs-readahead.rules <<EOM
+SUBSYSTEM=="bdi",
+ACTION=="add",
+PROGRAM="/usr/bin/awk -v bdi=$kernel 'BEGIN{ret=1} {if ($4 == bdi) {ret=0}} END{exit ret}' /proc/fs/nfsfs/volumes",
+ATTR{read_ahead_kb}="15380"
+EOM
+
+udevadm control --reload

mariner/common/install_amd_libs.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -ex
+
+# Set AOCC version
+amd_metadata=$(jq -r '.amd."'"$DISTRIBUTION"'"' <<< $COMPONENT_VERSIONS)
+AOCC_VERSION=$(jq -r '.aocc.version' <<< $amd_metadata)
+AOCC_SHA256=$(jq -r '.aocc.sha256' <<< $amd_metadata)
+
+# install dependency
+PACKAGE="aocc-compiler-${AOCC_VERSION}.x86_64.rpm"
+AOCC_DOWNLOAD_URL=https://download.amd.com/developer/eula/aocc-compiler/$PACKAGE
+$COMMON_DIR/download_and_verify.sh $AOCC_DOWNLOAD_URL $AOCC_SHA256 
+tdnf install -y ./$PACKAGE
+
+rm ./$PACKAGE
+$COMMON_DIR/write_component_version.sh "AOCC" ${AOCC_VERSION}

mariner/common/install_intel_libs.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -ex
+
+# Set Intel® oneAPI Math Kernel Library info
+intel_one_mkl_metadata=$(jq -r '.intel_one_mkl."'"$DISTRIBUTION"'"' <<< $COMPONENT_VERSIONS)
+INTEL_ONE_MKL_VERSION=$(jq -r '.version' <<< $intel_one_mkl_metadata)
+INTEL_ONE_MKL_SHA256=$(jq -r '.sha256' <<< $intel_one_mkl_metadata)
+INTEL_ONE_MKL_DOWNLOAD_URL=$(jq -r '.url' <<< $intel_one_mkl_metadata)
+INTEL_ONE_MKL_OFFLINE_INSTALLER=$(basename $INTEL_ONE_MKL_DOWNLOAD_URL)
+
+# Intel® oneAPI Math Kernel Library
+$COMMON_DIR/write_component_version.sh "INTEL_ONE_MKL" ${INTEL_ONE_MKL_VERSION}
+$COMMON_DIR/download_and_verify.sh ${INTEL_ONE_MKL_DOWNLOAD_URL} ${INTEL_ONE_MKL_SHA256}
+sh ./${INTEL_ONE_MKL_OFFLINE_INSTALLER} -s -a -s --eula accept

mariner/common/install_lustre_client.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -ex
+
+# Set Lustre driver version
+lustre_version=$(jq -r '.lustre."'"$DISTRIBUTION"'".version' <<< $COMPONENT_VERSIONS)
+
+# Expected params:
+# $1 = the major version of the distro. "8" for RHEL/Alma8, "9" for RHEL/Alma9.
+
+source $MARINER_COMMON_DIR/setup_lustre_repo.sh "$1"
+
+tdnf install -y --disableexcludes=main --refresh amlfs-lustre-client-$lustre_version-$(uname -r | sed -e "s/\.$(uname -p)$//" | sed -re 's/[-_]/\./g')-1
+sed -i "$ s/$/ amlfs*/" /etc/dnf/dnf.conf
+
+$COMMON_DIR/write_component_version.sh "lustre" $lustre_version

mariner/common/install_monitoring_tools.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e
+
+# Set moneo metadata
+MONEO_VERSION=$(jq -r '.moneo."'"$DISTRIBUTION"'".version' <<< $COMPONENT_VERSIONS)
+
+# Dependencies 
+python3 -m pip install --upgrade pip
+
+# Adding path to sudo user
+sed -i 's/.*secure_path.*/Defaults    secure_path = "\/usr\/local\/sbin:\/usr\/local\/bin:\/sbin:\/bin:\/usr\/sbin:\/usr\/bin\/"/' /etc/sudoers
+
+MONITOR_DIR=/opt/azurehpc/tools
+
+mkdir -p $MONITOR_DIR
+
+pushd $MONITOR_DIR
+    git clone https://github.com/Azure/Moneo  --branch v$MONEO_VERSION
+
+    chmod 777 Moneo
+
+    pushd Moneo/linux_service
+        ./configure_service.sh    
+    popd
+popd
+
+# add an alias for Moneo
+if ! grep -qxF "alias moneo='python3 /opt/azurehpc/tools/Moneo/moneo.py'" /etc/bashrc; then # TODO: bashrc or bash.bashrc? 
+    echo "alias moneo='python3 /opt/azurehpc/tools/Moneo/moneo.py'" >> /etc/bashrc
+fi
+
+$COMMON_DIR/write_component_version.sh "MONEO" ${MONEO_VERSION}