configure-ovs-network.yaml

mode: 0755
path: "/usr/local/bin/configure-ovs.sh"
contents:
  inline: |
    #!/bin/bash
    set -eux
    # This file is not needed anymore in 4.7+, but when rolling back to 4.6
    # the ovs pod needs it to know ovs is running on the host.
    touch /var/run/ovs-config-executed

    NM_CONN_OVERLAY="/etc/NetworkManager/systemConnectionsMerged"
    NM_CONN_UNDERLAY="/etc/NetworkManager/system-connections"
    if [ -d "$NM_CONN_OVERLAY" ]; then
      NM_CONN_PATH="$NM_CONN_OVERLAY"
    else
      NM_CONN_PATH="$NM_CONN_UNDERLAY"
    fi

    MANAGED_NM_CONN_SUFFIX="-slave-ovs-clone"
    # Workaround to ensure OVS is installed due to bug in systemd Requires:
    # https://bugzilla.redhat.com/show_bug.cgi?id=1888017
    copy_nm_conn_files() {
      local src_path="$NM_CONN_PATH"
      local dst_path="$1"
      if [ "$src_path" = "$dst_path" ]; then
        echo "No need to copy configuration files, source and destination are the same"
        return
      fi
      if [ -d "$src_path" ]; then
        echo "$src_path exists"
        local files=("${MANAGED_NM_CONN_FILES[@]}")
        shopt -s nullglob
        files+=($src_path/*${MANAGED_NM_CONN_SUFFIX}.nmconnection $src_path/*${MANAGED_NM_CONN_SUFFIX})
        shopt -u nullglob
        for file in "${files[@]}"; do
          file="$(basename $file)"
          if [ -f "$src_path/$file" ]; then
            if [ ! -f "$dst_path/$file" ]; then
              echo "Copying configuration $file"
              cp "$src_path/$file" "$dst_path/$file"
            elif ! cmp --silent "$src_path/$file" "$dst_path/$file"; then
              echo "Copying updated configuration $file"
              cp -f "$src_path/$file" "$dst_path/$file"
            else
              echo "Skipping $file since it's equal at destination"
            fi
          else
            echo "Skipping $file since it does not exist at source"
          fi
        done
      fi
    }

    persist_nm_conn_files() {
      copy_nm_conn_files "$NM_CONN_UNDERLAY"
    }

    update_nm_conn_files() {
      bridge_name=${1}
      port_name=${2}
      ovs_port="ovs-port-${bridge_name}"
      ovs_interface="ovs-if-${bridge_name}"
      default_port_name="ovs-port-${port_name}" # ovs-port-phys0
      bridge_interface_name="ovs-if-${port_name}" # ovs-if-phys0
    # In RHEL7 files in /{etc,run}/NetworkManager/system-connections end without the suffix '.nmconnection', whereas in RHCOS they end with the suffix.
      MANAGED_NM_CONN_FILES=($(echo {"$bridge_name","$ovs_interface","$ovs_port","$bridge_interface_name","$default_port_name"} {"$bridge_name","$ovs_interface","$ovs_port","$bridge_interface_name","$default_port_name"}.nmconnection))
    }

    # Used to remove files managed by configure-ovs
    rm_nm_conn_files() {
      local files=("${MANAGED_NM_CONN_FILES[@]}")
      shopt -s nullglob
      files+=(${NM_CONN_PATH}/*${MANAGED_NM_CONN_SUFFIX}.nmconnection ${NM_CONN_PATH}/*${MANAGED_NM_CONN_SUFFIX})
      shopt -u nullglob
      for file in "${files[@]}"; do
        file="$(basename $file)"
        # Also remove files in underlay
        for path in "${NM_CONN_PATH}" "${NM_CONN_UNDERLAY}"; do
          file_path="${path}/$file"
          if [ -f "$file_path" ]; then
            rm -f "$file_path"
            echo "Removed nmconnection file $file_path"
            nm_conn_files_removed=1
          fi
        done
      done
    }

    # Used to clone a slave connection by uuid, returns new uuid
    clone_slave_connection() {
      local uuid="$1"
      local old_name
      old_name="$(nmcli -g connection.id connection show uuid "$uuid")"
      local new_name="${old_name}${MANAGED_NM_CONN_SUFFIX}"
      if nmcli connection show id "${new_name}" &> /dev/null; then
        echo "WARN: existing ovs slave ${new_name} connection profile file found, overwriting..." >&2
        nmcli connection delete id "${new_name}" &> /dev/null
      fi
      nmcli connection clone $uuid "${new_name}" &> /dev/null
      nmcli -g connection.uuid connection show "${new_name}"
    }

    # Used to replace an old master connection uuid with a new one on all connections
    replace_connection_master() {
      local old="$1"
      local new="$2"
      for conn_uuid in $(nmcli -g UUID connection show) ; do
        if [ "$(nmcli -g connection.master connection show uuid "$conn_uuid")" != "$old" ]; then
          continue
        fi

        # make changes for slave profiles in a new clone
        local new_uuid
        new_uuid=$(clone_slave_connection $conn_uuid)

        nmcli conn mod uuid $new_uuid connection.master "$new"
        nmcli conn mod $new_uuid connection.autoconnect-priority 100
        echo "Replaced master $old with $new for slave profile $new_uuid"
      done
    }

    # Given an interface, generates NM configuration to add to an OVS bridge
    convert_to_bridge() {
      iface=${1}
      bridge_name=${2}
      port_name=${3}
      ovs_port="ovs-port-${bridge_name}"
      ovs_interface="ovs-if-${bridge_name}"
      default_port_name="ovs-port-${port_name}" # ovs-port-phys0
      bridge_interface_name="ovs-if-${port_name}" # ovs-if-phys0

      if [ "$iface" = "$bridge_name" ]; then
        # handle vlans and bonds etc if they have already been
        # configured via nm key files and br-ex is already up
        ifaces=$(ovs-vsctl list-ifaces ${iface})
        for intf in $ifaces; do configure_driver_options $intf; done
        echo "Networking already configured and up for ${bridge-name}!"
        return
      fi

      if [ -z "$iface" ]; then
        echo "ERROR: Unable to find default gateway interface"
        exit 1
      fi
      # find the MAC from OVS config or the default interface to use for OVS internal port
      # this prevents us from getting a different DHCP lease and dropping connection
      if ! iface_mac=$(<"/sys/class/net/${iface}/address"); then
        echo "Unable to determine default interface MAC"
        exit 1
      fi

      echo "MAC address found for iface: ${iface}: ${iface_mac}"

      # find MTU from original iface
      iface_mtu=$(ip link show "$iface" | awk '{print $5; exit}')
      if [[ -z "$iface_mtu" ]]; then
        echo "Unable to determine default interface MTU, defaulting to 1500"
        iface_mtu=1500
      else
        echo "MTU found for iface: ${iface}: ${iface_mtu}"
      fi

      # store old conn for later
      old_conn=$(nmcli --fields UUID,DEVICE conn show --active | awk "/\s${iface}\s*\$/ {print \$1}")

      extra_brex_args=""
      # check for dhcp client ids
      dhcp_client_id=$(nmcli --get-values ipv4.dhcp-client-id conn show ${old_conn})
      if [ -n "$dhcp_client_id" ]; then
        extra_brex_args+="ipv4.dhcp-client-id ${dhcp_client_id} "
      fi

      dhcp6_client_id=$(nmcli --get-values ipv6.dhcp-duid conn show ${old_conn})
      if [ -n "$dhcp6_client_id" ]; then
        extra_brex_args+="ipv6.dhcp-duid ${dhcp6_client_id} "
      fi

      ipv6_addr_gen_mode=$(nmcli --get-values ipv6.addr-gen-mode conn show ${old_conn})
      if [ -n "$ipv6_addr_gen_mode" ]; then
        extra_brex_args+="ipv6.addr-gen-mode ${ipv6_addr_gen_mode} "
      fi

      # create bridge; use NM's ethernet device default route metric (100)
      if ! nmcli connection show "$bridge_name" &> /dev/null; then
        nmcli c add type ovs-bridge \
            con-name "$bridge_name" \
            conn.interface "$bridge_name" \
            802-3-ethernet.mtu ${iface_mtu} \
            ipv4.route-metric 100 \
            ipv6.route-metric 100 \
            ${extra_brex_args}
      fi

      # find default port to add to bridge
      if ! nmcli connection show "$default_port_name" &> /dev/null; then
        nmcli c add type ovs-port conn.interface ${iface} master "$bridge_name" con-name "$default_port_name"
      fi

      if ! nmcli connection show "$ovs_port" &> /dev/null; then
        nmcli c add type ovs-port conn.interface "$bridge_name" master "$bridge_name" con-name "$ovs_port"
      fi

      extra_phys_args=()
      # check if this interface is a vlan, bond, team, or ethernet type
      if [ $(nmcli --get-values connection.type conn show ${old_conn}) == "vlan" ]; then
        iface_type=vlan
        vlan_id=$(nmcli --get-values vlan.id conn show ${old_conn})
        if [ -z "$vlan_id" ]; then
          echo "ERROR: unable to determine vlan_id for vlan connection: ${old_conn}"
          exit 1
        fi
        vlan_parent=$(nmcli --get-values vlan.parent conn show ${old_conn})
        if [ -z "$vlan_parent" ]; then
          echo "ERROR: unable to determine vlan_parent for vlan connection: ${old_conn}"
          exit 1
        fi
        extra_phys_args=( dev "${vlan_parent}" id "${vlan_id}" )
      elif [ $(nmcli --get-values connection.type conn show ${old_conn}) == "bond" ]; then
        iface_type=bond
        # check bond options
        bond_opts=$(nmcli --get-values bond.options conn show ${old_conn})
        if [ -n "$bond_opts" ]; then
          extra_phys_args+=( bond.options "${bond_opts}" )
        fi
      elif [ $(nmcli --get-values connection.type conn show ${old_conn}) == "team" ]; then
        iface_type=team
        # check team config options
        team_config_opts=$(nmcli --get-values team.config -e no conn show ${old_conn})
        if [ -n "$team_config_opts" ]; then
          # team.config is json, remove spaces to avoid problems later on
          extra_phys_args+=( team.config "${team_config_opts//[[:space:]]/}" )
        fi
      else
        iface_type=802-3-ethernet
      fi

      # use ${extra_phys_args[@]+"${extra_phys_args[@]}"} instead of ${extra_phys_args[@]} to be compatible with bash 4.2 in RHEL7.9
      if ! nmcli connection show "$bridge_interface_name" &> /dev/null; then
        nmcli c add type ${iface_type} conn.interface ${iface} master "$default_port_name" con-name "$bridge_interface_name" \
        connection.autoconnect-priority 100 802-3-ethernet.mtu ${iface_mtu} ${extra_phys_args[@]+"${extra_phys_args[@]}"}
      fi

      # Get the new connection uuid
      new_conn=$(nmcli -g connection.uuid conn show "$bridge_interface_name")

      # Update connections with master property set to use the new connection
      replace_connection_master $old_conn $new_conn
      replace_connection_master $iface $new_conn

      if ! nmcli connection show "$ovs_interface" &> /dev/null; then
        if nmcli --fields ipv4.method,ipv6.method conn show $old_conn | grep manual; then
          echo "Static IP addressing detected on default gateway connection: ${old_conn}"
          # find and copy the old connection to get the address settings
          if egrep -l "^uuid=$old_conn" ${NM_CONN_PATH}/*; then
            old_conn_file=$(egrep -l "^uuid=$old_conn" ${NM_CONN_PATH}/*)
            cloned=false
          else
            echo "WARN: unable to find NM configuration file for conn: ${old_conn}. Attempting to clone conn"
            nmcli conn clone ${old_conn} ${old_conn}-clone
            shopt -s nullglob
            old_conn_files=(${NM_CONN_PATH}/"${old_conn}"-clone*)
            shopt -u nullglob
            if [ ${#old_conn_files[@]} -ne 1 ]; then
              echo "ERROR: unable to locate cloned conn file for ${old_conn}-clone"
              exit 1
            fi
            old_conn_file="${old_conn_files[0]}"
            cloned=true
            echo "Successfully cloned conn to ${old_conn_file}"
          fi
          echo "old connection file found at: ${old_conn_file}"
          old_basename=$(basename "${old_conn_file}" .nmconnection)
          new_conn_file="${old_conn_file/${NM_CONN_PATH}\/$old_basename/${NM_CONN_PATH}/$ovs_interface}"
          if [ -f "$new_conn_file" ]; then
            echo "WARN: existing $bridge_name interface file found: $new_conn_file, which is not loaded in NetworkManager...overwriting"
          fi
          cp -f "${old_conn_file}" ${new_conn_file}
          restorecon ${new_conn_file}
          if $cloned; then
            nmcli conn delete ${old_conn}-clone
            rm -f "${old_conn_file}"
          fi
          ovs_port_conn=$(nmcli --fields connection.uuid conn show "$ovs_port" | awk '{print $2}')
          br_iface_uuid=$(cat /proc/sys/kernel/random/uuid)
          # modify file to work with OVS and have unique settings
          sed -i '/^\[connection\]$/,/^\[/ s/^uuid=.*$/uuid='"$br_iface_uuid"'/' ${new_conn_file}
          sed -i '/^multi-connect=.*$/d' ${new_conn_file}
          sed -i '/^\[connection\]$/,/^\[/ s/^type=.*$/type=ovs-interface/' ${new_conn_file}
          sed -i '/^\[connection\]$/,/^\[/ s/^id=.*$/id='"$ovs_interface"'/' ${new_conn_file}
          sed -i '/^\[connection\]$/a slave-type=ovs-port' ${new_conn_file}
          sed -i '/^\[connection\]$/a master='"$ovs_port_conn" ${new_conn_file}
          if grep 'interface-name=' ${new_conn_file} &> /dev/null; then
            sed -i '/^\[connection\]$/,/^\[/ s/^interface-name=.*$/interface-name='"$bridge_name"'/' ${new_conn_file}
          else
            sed -i '/^\[connection\]$/a interface-name='"$bridge_name" ${new_conn_file}
          fi
          if ! grep 'cloned-mac-address=' ${new_conn_file} &> /dev/null; then
            sed -i '/^\[ethernet\]$/a cloned-mac-address='"$iface_mac" ${new_conn_file}
          else
            sed -i '/^\[ethernet\]$/,/^\[/ s/^cloned-mac-address=.*$/cloned-mac-address='"$iface_mac"'/' ${new_conn_file}
          fi
          if grep 'mtu=' ${new_conn_file} &> /dev/null; then
            sed -i '/^\[ethernet\]$/,/^\[/ s/^mtu=.*$/mtu='"$iface_mtu"'/' ${new_conn_file}
          else
            sed -i '/^\[ethernet\]$/a mtu='"$iface_mtu" ${new_conn_file}
          fi
          cat <<EOF >> ${new_conn_file}
    [ovs-interface]
    type=internal
    EOF
          nmcli c load ${new_conn_file}
          echo "Loaded new $ovs_interface connection file: ${new_conn_file}"
        else
          nmcli c add type ovs-interface slave-type ovs-port conn.interface "$bridge_name" master "$ovs_port" con-name \
            "$ovs_interface" 802-3-ethernet.mtu ${iface_mtu} 802-3-ethernet.cloned-mac-address ${iface_mac} \
            ipv4.route-metric 100 ipv6.route-metric 100
        fi
      fi

      configure_driver_options "${iface}"
      update_nm_conn_files "$bridge_name" "$port_name"
      persist_nm_conn_files
    }

    # Used to remove a bridge
    remove_ovn_bridges() {
      bridge_name=${1}
      port_name=${2}

      # Reload configuration, after reload the preferred connection profile
      # should be auto-activated
      update_nm_conn_files ${bridge_name} ${port_name}
      rm_nm_conn_files

      # NetworkManager will not remove ${bridge_name} if it has the patch port created by ovn-kubernetes
      # so remove explicitly
      ovs-vsctl --timeout=30 --if-exists del-br ${bridge_name}
    }

    # Removes any previous ovs configuration
    remove_all_ovn_bridges() {
      echo "Reverting any previous OVS configuration"
      
      remove_ovn_bridges br-ex phys0
      if [ -d "/sys/class/net/br-ex1" ]; then
        remove_ovn_bridges br-ex1 phys1
      fi
      
      echo "OVS configuration successfully reverted"
    }

    # Reloads NetworkManager
    reload_nm() {
      echo "Reloading NetworkManager..."

      # set network off, so that auto-connect priority is evaluated when turning
      # it back on
      nmcli network off
      
      # restart NetworkManager to reload profiles, including generating
      # transient profiles for devices that don't have any
      systemctl restart NetworkManager

      # turn network back on triggering auto-connects 
      nmcli network on

      # Wait until all profiles auto-connect
      if nm-online -s -t 60; then
        echo "NetworkManager has activated all suitable profiles after reload"
      else
        echo "NetworkManager has not activated all suitable profiles after reload"
      fi

      # Check if we have any type of connectivity
      if nm-online -t 0; then
        echo "NetworkManager has connectivity after reload"
      else
        echo "NetworkManager does not have connectivity after reload"
      fi
    }

    # Activates a NM connection profile
    activate_nm_conn() {
      local conn="$1"
      for i in {1..10}; do
        echo "Attempt $i to bring up connection $conn"
        nmcli conn up "$conn" && s=0 && break || s=$?
        sleep 5
      done
      if [ $s -eq 0 ]; then
        echo "Brought up connection $conn successfully"
      else
        echo "ERROR: Cannot bring up connection $conn after $i attempts"
      fi
      return $s
    }

    # Used to print network state
    print_state() {
      echo "Current connection, interface and routing state:"
      nmcli -g all c show
      ip -d address show
      ip route show
      ip -6 route show
    }

    # Setup an exit trap to rollback on error
    handle_exit() {
      e=$?
      [ $e -eq 0 ] && print_state && exit 0

      echo "ERROR: configure-ovs exited with error: $e"
      print_state

      # copy configuration to tmp
      dir=$(mktemp -d -t "configure-ovs-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX")
      update_nm_conn_files br-ex phys0
      copy_nm_conn_files "$dir"
      update_nm_conn_files br-ex1 phys1
      copy_nm_conn_files "$dir"
      echo "Copied OVS configuration to $dir for troubleshooting"

      # attempt to restore the previous network state
      echo "Attempting to restore previous configuration..."
      remove_all_ovn_bridges
      reload_nm
      print_state

      exit $e
    }
    trap "handle_exit" EXIT

    if ! rpm -qa | grep -q openvswitch; then
      echo "Warning: Openvswitch package is not installed!"
      exit 1
    fi

    # print initial state
    print_state

    if [ "$1" == "OVNKubernetes" ]; then
      # Configures NICs onto OVS bridge "br-ex"
      # Configuration is either auto-detected or provided through a config file written already in Network Manager
      # key files under /etc/NetworkManager/system-connections/
      # Managing key files is outside of the scope of this script

      # if the interface is of type vmxnet3 add multicast capability for that driver
      # REMOVEME: Once BZ:1854355 is fixed, this needs to get removed.
      function configure_driver_options {
        intf=$1
        if [ ! -f "/sys/class/net/${intf}/device/uevent" ]; then
          echo "Device file doesn't exist, skipping setting multicast mode"
        else
          driver=$(cat "/sys/class/net/${intf}/device/uevent" | grep DRIVER | awk -F "=" '{print $2}')
          echo "Driver name is" $driver
          if [ "$driver" = "vmxnet3" ]; then
            ifconfig "$intf" allmulti
          fi
        fi
      }

      # on every boot we rollback and generate the configuration again, to take
      # in any changes that have possibly been applied in the standard
      # configuration sources
      if [ ! -f /run/configure-ovs-boot-done ]; then
        echo "Running on boot, restoring to previous configuration..."
        remove_all_ovn_bridges
        reload_nm
        print_state
      fi
      touch /run/configure-ovs-boot-done

      iface=""
      counter=0
      # find default interface
      while [ $counter -lt 12 ]; do
        # check ipv4
        iface=$(ip route show default | awk '{ if ($4 == "dev") { print $5; exit } }')
        if [[ -n "$iface" ]]; then
          echo "IPv4 Default gateway interface found: ${iface}"
          break
        fi
        # check ipv6
        iface=$(ip -6 route show default | awk '{ if ($4 == "dev") { print $5; exit } }')
        if [[ -n "$iface" ]]; then
          echo "IPv6 Default gateway interface found: ${iface}"
          break
        fi
        counter=$((counter+1))
        echo "No default route found on attempt: ${counter}"
        sleep 5
      done

      extra_bridge_file='/etc/ovnk/extra_bridge'

      # Some deployments uses a temporary solution where br-ex is moved out from the default gateway interface
      # and bound to a different nic. If that is the case, we rollback (https://github.com/trozet/openshift-ovn-migration).
      if [ "$iface" != "br-ex" ] && [  -f "$extra_bridge_file" ] && nmcli connection show br-ex &> /dev/null; then
        echo "Default gateway is not bridge but bridge exists, reverting"
        remove_all_ovn_bridges
        reload_nm
        print_state
      fi

      convert_to_bridge "$iface" "br-ex" "phys0"

      # Check if we need to configure the second bridge
      if [ -f "$extra_bridge_file" ] && (! nmcli connection show br-ex1 &> /dev/null || ! nmcli connection show ovs-if-phys1 &> /dev/null); then
        interface=$(head -n 1 $extra_bridge_file)
        convert_to_bridge "$interface" "br-ex1" "phys1"
      fi

      # Check if we need to remove the second bridge
      if [ ! -f "$extra_bridge_file" ] && (nmcli connection show br-ex1 &> /dev/null || nmcli connection show ovs-if-phys1 &> /dev/null); then
        update_nm_conn_files br-ex1 phys1
        rm_nm_conn_files
      fi

      # Remove bridges created by openshift-sdn
      ovs-vsctl --timeout=30 --if-exists del-br br0
      
      # Recycle NM connections
      reload_nm

      # Make sure everything is activated
      activate_nm_conn ovs-if-phys0
      activate_nm_conn ovs-if-br-ex
      if [ -f "$extra_bridge_file" ]; then
        activate_nm_conn ovs-if-phys1
        activate_nm_conn ovs-if-br-ex1
      fi
    elif [ "$1" == "OpenShiftSDN" ]; then
      # This will be set to 1 if remove_all_ovn_bridges actually changes anything
      nm_conn_files_removed=0

      # Revert changes made by /usr/local/bin/configure-ovs.sh during SDN migration.
      remove_all_ovn_bridges
      
      # Reload only if we removed connection profiles
      if [ $nm_conn_files_removed -eq 1 ]; then
        echo "OVS configuration was cleaned up, will reload NetworkManager"
        reload_nm
      fi
      
      # Remove bridges created by ovn-kubernetes
      ovs-vsctl --timeout=30 --if-exists del-br br-int -- --if-exists del-br br-local
    fi