Skip to content

Commit

Permalink
DPDK: Allow arbitrary rdma-core source build
Browse files Browse the repository at this point in the history 
Fulfilling an ask to be able to test multiple rdma-core
versions to allow development and regression tests.

Setting rdma_core_source to either a git or tar.gz link allows you to
build from source. For git either set rdma_core_ref or it will pick the
latest version sorted tag.
  • Loading branch information
@mcgov
mcgov committed Dec 20, 2023
1 parent 44c42cf commit cfe22b9
Show file tree
Hide file tree
Showing 4 changed files with 293 additions and 94 deletions.
61 changes: 58 additions & 3 deletions lisa/util/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
Union,
cast,
)
from urllib.parse import urlparse

import paramiko
import pluggy
Expand All @@ -47,7 +48,7 @@
# source -
# https://github.com/django/django/blob/stable/1.3.x/django/core/validators.py#L45
__url_pattern = re.compile(
r"^(?:http|ftp)s?://" # http:// or https://
r"^(?:http|s?ftp)s?://" # http:// or https://
r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)"
r"+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|" # ...domain
r"localhost|" # localhost...
Expand Down Expand Up @@ -563,7 +564,9 @@ def find_group_in_lines(
return result


def deep_update_dict(src: Dict[str, Any], dest: Dict[str, Any]) -> Dict[str, Any]:
def deep_update_dict(
src: Dict[str, Any], dest: Optional[Dict[str, Any]]
) -> Dict[str, Any]:
if (
dest is None
or isinstance(dest, int)
Expand All @@ -577,7 +580,7 @@ def deep_update_dict(src: Dict[str, Any], dest: Dict[str, Any]) -> Dict[str, Any

if isinstance(result, dict):
for key, value in src.items():
if isinstance(value, dict) and key in dest:
if dest and isinstance(value, dict) and key in dest:
value = deep_update_dict(value, dest[key])
result[key] = value
elif isinstance(src, dict):
Expand All @@ -598,6 +601,58 @@ def is_valid_url(url: str, raise_error: bool = True) -> bool:
return is_url


def is_valid_source_code_package(
source_url: str,
expected_package_name_pattern: Pattern[str],
allowed_protocols: Optional[List[str]] = None,
expected_domains: Optional[List[str]] = None,
) -> bool:
# avoid using a mutable default parameter
if not allowed_protocols:
allowed_protocols = [
"https",
"sftp",
]
# first, check if it's a url.
if not is_valid_url(url=source_url, raise_error=False):
return False

# NOTE: urllib might not work as you'd expect.
# It doesn't throw on lots of things you wouldn't expect to be urls.
# You must verify the parts on your own, some of them may be empty, some null.
# check: https://docs.python.org/3/library/urllib.parse.html#url-parsing

try:
parts = urlparse(source_url)
except ValueError:
return False

# ex: from https://www.com/path/to/file.tar
# scheme : https
# netloc : www.com
# path : path/to/file.tar

# get the filename from the path portion of the url
file_path = parts.path.split("/")[-1]
full_match = expected_package_name_pattern.match(file_path)
if not full_match:
return False

# check the expected domain is correct if present
valid_netloc = not expected_domains or any(
[domain.endswith(parts.netloc) for domain in expected_domains]
)

# optional but default is check access is via sftp/https
valid_scheme = any([parts.scheme == x for x in allowed_protocols])
return (
valid_scheme
and parts.netloc != ""
and valid_netloc
and (full_match.group(0) == file_path)
)


def filter_ansi_escape(content: str) -> str:
return __ansi_escape.sub("", content)

Expand Down
139 changes: 48 additions & 91 deletions microsoft/testsuites/dpdk/dpdktestpmd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
Kill,
Lscpu,
Lspci,
Make,
Modprobe,
Pidof,
Pkgconfig,
Expand All @@ -40,6 +39,7 @@
is_ubuntu_latest_or_prerelease,
is_ubuntu_lts_version,
)
from microsoft.testsuites.dpdk.rdma_core import RdmaCoreManager

PACKAGE_MANAGER_SOURCE = "package_manager"

Expand Down Expand Up @@ -136,22 +136,6 @@ def command(self) -> str:
_rx_pps_key: r"Rx-pps:\s+([0-9]+)",
}

def get_rdma_core_package_name(self) -> str:
distro = self.node.os
package = ""
# check if rdma-core is installed already...
if self.node.tools[Pkgconfig].package_info_exists("libibuverbs"):
return package
if isinstance(distro, Debian):
package = "rdma-core ibverbs-providers libibverbs-dev"
elif isinstance(distro, Suse):
package = "rdma-core-devel librdmacm1"
elif isinstance(distro, Fedora):
package = "librdmacm-devel"
else:
fail("Invalid OS for rdma-core source installation.")
return package

@property
def can_install(self) -> bool:
for _os in [Debian, Fedora, Suse]:
Expand Down Expand Up @@ -465,8 +449,17 @@ def add_sample_apps_to_build_list(self, apps: Union[List[str], None]) -> None:

def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
# set source args for builds if needed, first for dpdk
self._dpdk_source = kwargs.pop("dpdk_source", PACKAGE_MANAGER_SOURCE)
self._dpdk_branch = kwargs.pop("dpdk_branch", "main")
# then for rdma-core
rdma_core_source = kwargs.pop("rdma_core_source", "")
rdma_core_ref = kwargs.pop("rdma_core_ref", "")
self.rdma_core = RdmaCoreManager(
node=self.node,
rdma_core_source=rdma_core_source,
rdma_core_ref=rdma_core_ref,
)
self._sample_apps_to_build = kwargs.pop("sample_apps", [])
self._dpdk_version_info = VersionInfo(0, 0)
self._testpmd_install_path: str = ""
Expand Down Expand Up @@ -522,54 +515,6 @@ def _check_pps_data(self, rx_or_tx: str) -> None:
f"empty or all zeroes for dpdktestpmd.{rx_or_tx.lower()}_pps_data."
).is_true()

def _install_upstream_rdma_core_for_mana(self) -> None:
node = self.node
wget = node.tools[Wget]
make = node.tools[Make]
tar = node.tools[Tar]
distro = node.os

if isinstance(distro, Debian):
distro.install_packages(
"cmake libudev-dev "
"libnl-3-dev libnl-route-3-dev ninja-build pkg-config "
"valgrind python3-dev cython3 python3-docutils pandoc "
"libssl-dev libelf-dev python3-pip libnuma-dev"
)
elif isinstance(distro, Fedora):
distro.group_install_packages("Development Tools")
distro.install_packages(
"cmake gcc libudev-devel "
"libnl3-devel pkg-config "
"valgrind python3-devel python3-docutils "
"openssl-devel unzip "
"elfutils-devel python3-pip libpcap-devel "
"tar wget dos2unix psmisc kernel-devel-$(uname -r) "
"librdmacm-devel libmnl-devel kernel-modules-extra numactl-devel "
"kernel-headers elfutils-libelf-devel meson ninja-build libbpf-devel "
)
else:
# check occcurs before this function
return

tar_path = wget.get(
url=(
"https://github.com/linux-rdma/rdma-core/"
"releases/download/v46.0/rdma-core-46.0.tar.gz"
),
file_path=str(node.working_path),
)

tar.extract(tar_path, dest_dir=str(node.working_path), gzip=True, sudo=True)
source_path = node.working_path.joinpath("rdma-core-46.0")
node.execute(
"cmake -DIN_PLACE=0 -DNO_MAN_PAGES=1 -DCMAKE_INSTALL_PREFIX=/usr",
shell=True,
cwd=source_path,
sudo=True,
)
make.make_install(source_path)

def _set_backport_repo_args(self) -> None:
distro = self.node.os
# skip attempting to use backports for latest/prerlease
Expand Down Expand Up @@ -597,6 +542,18 @@ def _install(self) -> bool:
self._testpmd_output_during_rescind = ""
self._last_run_output = ""
node = self.node
distro = node.os
if not (
isinstance(distro, Fedora)
or isinstance(distro, Debian)
or isinstance(distro, Suse)
):
raise SkippedException(
UnsupportedDistroException(
distro, "DPDK tests not implemented for this OS."
)
)

# before doing anything: determine if backport repo needs to be enabled
self._set_backport_repo_args()

Expand All @@ -608,42 +565,50 @@ def _install(self) -> bool:
self._load_drivers_for_dpdk()
return True

# otherwise, install from package manager, git, or tar

self._install_dependencies()
# if this is mana VM, we don't support other distros yet
is_mana_test_supported = isinstance(distro, Ubuntu) or isinstance(
distro, Fedora
)
if self.is_mana and not is_mana_test_supported:
raise SkippedException("MANA DPDK test is not supported on this OS")

# if this is mana VM, we need an upstream rdma-core package (for now)
if self.is_mana:
if not (isinstance(node.os, Ubuntu) or isinstance(node.os, Fedora)):
raise SkippedException("MANA DPDK test is not supported on this OS")
# if we need an rdma-core source install, do it now.
if self.rdma_core.can_install_from_source() or (
is_mana_test_supported and self.is_mana
):
# ensure no older version is installed
distro.uninstall_packages("rdma-core")
self.rdma_core.do_source_install()

# ensure no older dependency is installed
node.os.uninstall_packages("rdma-core")
self._install_upstream_rdma_core_for_mana()
# otherwise, install kernel and dpdk deps from package manager, git, or tar
self._install_dependencies()
# install any missing rdma-core packages
rdma_packages = self.rdma_core.get_missing_distro_packages()
distro.install_packages(rdma_packages)

# installing from distro package manager
if self.use_package_manager_install():
self.node.log.info(
"Installing dpdk and dev package from package manager..."
)
if isinstance(node.os, Debian):
node.os.install_packages(
if isinstance(distro, Debian):
distro.install_packages(
["dpdk", "dpdk-dev"],
extra_args=self._backport_repo_args,
)
elif isinstance(node.os, (Fedora, Suse)):
node.os.install_packages(["dpdk", "dpdk-devel"])
elif isinstance(distro, (Fedora, Suse)):
distro.install_packages(["dpdk", "dpdk-devel"])
else:
raise NotImplementedError(
"Dpdk package names are missing in dpdktestpmd.install"
f" for os {node.os.name}"
f" for os {distro.name}"
)
self.node.log.info(
f"Installed DPDK version {str(self._dpdk_version_info)} "
"from package manager"
)

self._dpdk_version_info = node.os.get_package_information("dpdk")
self._dpdk_version_info = distro.get_package_information("dpdk")
self.find_testpmd_binary()
self._load_drivers_for_dpdk()
return True
Expand Down Expand Up @@ -867,9 +832,6 @@ def _install_suse_dependencies(self) -> None:
suse.install_packages(self._suse_packages)
if not self.use_package_manager_install():
self._install_ninja_and_meson()
rdma_core_packages = self.get_rdma_core_package_name()
if rdma_core_packages:
suse.install_packages(rdma_core_packages.split())

def _install_ubuntu_dependencies(self) -> None:
node = self.node
Expand Down Expand Up @@ -904,9 +866,6 @@ def _install_ubuntu_dependencies(self) -> None:
# MANA tests use linux-modules-extra-azure, install if it's available.
if self.is_mana and ubuntu.is_package_in_repo("linux-modules-extra-azure"):
ubuntu.install_packages("linux-modules-extra-azure")
rdma_core_packages = self.get_rdma_core_package_name()
if rdma_core_packages:
ubuntu.install_packages(rdma_core_packages.split())

def _install_fedora_dependencies(self) -> None:
node = self.node
Expand All @@ -919,7 +878,7 @@ def _install_fedora_dependencies(self) -> None:
return # appease the type checker

# DPDK is very sensitive to rdma-core/kernel mismatches
# update to latest kernel before instaling dependencies
# update to latest kernel before installing dependencies
rhel.install_packages("kernel")
node.reboot()

Expand All @@ -935,9 +894,7 @@ def _install_fedora_dependencies(self) -> None:

# RHEL 8 doesn't require special cases for installed packages.
# TODO: RHEL9 may require updates upon release
rdma_core_packages = self.get_rdma_core_package_name()
if rdma_core_packages:
self._fedora_packages += rdma_core_packages.split()
if not self.rdma_core.is_installed_from_source:
rhel.group_install_packages("Infiniband Support")

rhel.group_install_packages("Development Tools")
Expand Down
4 changes: 4 additions & 0 deletions microsoft/testsuites/dpdk/dpdkutil.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,8 @@ def initialize_node_resources(
_set_forced_source_by_distro(node, variables)
dpdk_source = variables.get("dpdk_source", PACKAGE_MANAGER_SOURCE)
dpdk_branch = variables.get("dpdk_branch", "")
rdma_core_source = variables.get("rdma_core_source", "")
rdma_core_ref = variables.get("rdma_core_git_ref", "")
force_net_failsafe_pmd = variables.get("dpdk_force_net_failsafe_pmd", False)
log.info(
"Dpdk initialize_node_resources running"
Expand Down Expand Up @@ -348,6 +350,8 @@ def initialize_node_resources(
dpdk_branch=dpdk_branch,
sample_apps=sample_apps,
force_net_failsafe_pmd=force_net_failsafe_pmd,
rdma_core_source=rdma_core_source,
rdma_core_ref=rdma_core_ref,
)

# init and enable hugepages (required by dpdk)
Expand Down
Loading

0 comments on commit cfe22b9

Please sign in to comment.