diff --git a/lisa/nic.py b/lisa/nic.py index 35e33f8912..21565d50e2 100644 --- a/lisa/nic.py +++ b/lisa/nic.py @@ -455,6 +455,9 @@ def is_mana_device_present(self) -> bool: def is_mana_driver_enabled(self) -> bool: return self._node.tools[KernelConfig].is_enabled("CONFIG_MICROSOFT_MANA") + def is_mana_ib_driver_enabled(self) -> bool: + return self._node.tools[KernelConfig].is_enabled("CONFIG_MANA_INFINIBAND") + def _get_default_nic(self) -> None: self.default_nic: str = "" self.default_nic_route: str = "" diff --git a/lisa/operating_system.py b/lisa/operating_system.py index 7f466ef589..0ea253d747 100644 --- a/lisa/operating_system.py +++ b/lisa/operating_system.py @@ -6,7 +6,8 @@ from dataclasses import dataclass from enum import Enum from functools import partial -from pathlib import Path +from pathlib import Path, PurePath +from threading import Lock from typing import ( TYPE_CHECKING, Any, @@ -52,6 +53,7 @@ get_matched_str, parse_version, retry_without_exceptions, + to_bool, ) from lisa.util.logger import get_logger from lisa.util.perf_timer import create_timer @@ -354,6 +356,16 @@ class Posix(OperatingSystem, BaseClassMixin): def __init__(self, node: Any) -> None: super().__init__(node, is_posix=True) self._first_time_installation: bool = True + # instance variables to check if source repositories are enabled + # some distros make this easier than others, + # but all allow use of source code packages which are compiled + # on the local machine instead of binary packages. + # slower, but allows local optimization and auditing. + # lucky for us! those packages contain build dependency metadata. + # we can use this to avoid keeping long lists + # of constantly changing package names for each distro. + self._source_installation_initialized = False + self._lock = Lock() @classmethod def type_name(cls) -> str: @@ -446,6 +458,32 @@ def is_package_in_repo(self, package: Union[str, Tool, Type[Tool]]) -> bool: self._first_time_installation = False return self._is_package_in_repo(package_name) + def enable_package_build_deps(self) -> None: + """ + Enable apt-get/yum build-deps package build dependency installation by + turning on deb-src or srpm repositories. + """ + self._enable_package_build_deps() + + def install_build_deps( + self, + packages: Union[str, Tool, Type[Tool], Sequence[Union[str, Tool, Type[Tool]]]], + ) -> None: + """ + Install apt-get/yum build-deps package build dependency installation by + turning on deb-src or srpm repositories. + """ + # lock this to prevent double initialization; + # some methods involve directly writing to a file + with self._lock: + if not self._source_installation_initialized: + self.enable_package_build_deps() + self._source_installation_initialized = True + + package_names = self._get_package_list(packages) + for package in package_names: + self._install_build_deps(package) + def update_packages( self, packages: Union[str, Tool, Type[Tool], Sequence[Union[str, Tool, Type[Tool]]]], @@ -562,6 +600,12 @@ def _uninstall_packages( def _update_packages(self, packages: Optional[List[str]] = None) -> None: raise NotImplementedError() + def _enable_package_build_deps(self) -> None: + raise NotImplementedError() + + def _install_build_deps(self, packages: str) -> None: + raise NotImplementedError() + def _package_exists(self, package: str) -> bool: raise NotImplementedError() @@ -775,29 +819,129 @@ def name_pattern(cls) -> Pattern[str]: return re.compile("^Alpine|alpine|alpaquita") +# from man sources.list +# THE DEB AND DEB-SRC TYPES: GENERAL FORMAT +# The deb type references a typical two-level Debian archive, distribution/component. +# The distribution is generally a suite name like stable or testing or a +# codename like bullseye or bookworm while component is one of main, contrib or non-free. +# The deb-src type references a Debian distribution's source code in the +# same form as the deb type. A deb-src line is required to fetch source indexes. + +# The format for two one-line-style entries using the deb and deb-src types is: + +# deb [ option1=value1 option2=value2 ] uri suite [component1] [component2] [...] +# deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [...] + +# Alternatively the equivalent entry in deb822 style looks like this: + + +# Types: deb deb-src +# URIs: uri +# Suites: suite +# Components: [component1] [component2] [...] +# option1: value1 +# option2: value2 +_deb822_sources_entry_pattern = re.compile( + r"Types:\s*(?P[\w\s\-]+)\s*" + r"URIs:\s*(?P\S+)\s*" + r"Suites:\s*(?P.*)\n" + r"Components:\s*(?P.+?)\s*" + r"(?P((?:\S+:\s*.+)\n?)*)$" + r"\n?", + re.MULTILINE | re.DOTALL, +) +_deb822_option_entry_pattern = re.compile(r"(?P\S+):\s+(?P.+)$") + + @dataclass -# `apt-get update` repolist is of the form `: ` # Example: -# Get:5 http://azure.archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [1298 kB] # noqa: E501 +# deb-src http://debian-archive.trafficmanager.net/debian-security bullseye-security main restricted class DebianRepositoryInfo(RepositoryInfo): - # status for the repository. Examples: `Hit`, `Get` - status: str + # type, deb or deb-src + type: str - # id for the repository. Examples : 1, 2 - id: str + # options, eg: arch=amd64 + options: str # uri for the repository. Example: `http://azure.archive.ubuntu.com/ubuntu` uri: str - # metadata for the repository. Example: `amd64 Packages [1298 kB]` - metadata: str + # suite eg: focal-updates + suite: str + + # copmonents eg: main restricted + components: str + + enabled: bool + + # note: params in constructor are out of order compared to their placement in the file + # required for optional python params coming last + def __init__( + self, + name: str, + type: str, + uri: str, + suite: str, + options: str = "", + components: str = "", + ) -> None: + super().__init__(name) + self.type = type + self.options = options + self.uri = uri + self.suite = suite + self.components = components + # deb822 subclass can set this, sources.list we'll just assume they're enabled. + # we won't parse commented lines in them + self.enabled = True + + def __str__(self) -> str: + return f"{self.type} {self.options} {self.uri} {self.suite} {self.components}" + + def source_repo_enabled(self) -> bool: + return "deb-src" in self.type and self.enabled + + +@dataclass +# Example: +# deb-src http://debian-archive.trafficmanager.net/debian-security bullseye-security main restricted +class DebianDeb822RepositoryInfo(DebianRepositoryInfo): + # deb822 lets you add an enabled field which will be respected. + enabled: bool + + # note: params in constructor are out of order compared to their placement in the file + # required for optional python params coming last + def __init__( + self, + name: str, + type: str, + uri: str, + suite: str, + options: str = "", + components: str = "", + enabled: bool = True, + ) -> None: + super().__init__(name, type, uri, suite, options, components) + self.enabled = enabled + + def __str__(self) -> str: + return ( + f"Types:{self.type}\n" + f"Suites:{self.uri}\n" + f"Suites:{self.suite}\n" + f"{self.components}" + ) class Debian(Linux): - # Get:5 http://azure.archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [1298 kB] # noqa: E501 + # see sources.list + # deb [ option1=value1 option2=value2 ] uri suite [component1] [component2] [...] _debian_repository_info_pattern = re.compile( - r"(?P\S+):(?P\d+)\s+(?P\S+)\s+(?P\S+)" - r"\s+(?P.*)\s*" + r"^(?Pdeb(?:-src)?)\s+" # deb or deb-src + r"(?P(?:\w+=\w+\s?)*)\s" # [option1=value1 ...] + r"(?P\S+)\s+" # uri + r"(?P\S+)\s+" # suite + r"(?P.*)$", # optional components ) # ex: 3.10 @@ -934,6 +1078,59 @@ def _get_package_information(self, package_name: str) -> LisaVersionInfo: ) return self._cache_and_return_version_info(package_name, version_info) + def _enable_package_build_deps(self) -> None: + # debian uses apt sources.list. + # we only need to uncomment the standard source repos + # in that file + node = self._node + repos = self.get_repositories() + codename = self.information.codename + # if [ + # repo + # for repo in repos + # if ( + # isinstance(repo, (DebianRepositoryInfo, DebianDeb822RepositoryInfo)) + # and repo.enabled + # and (repo.name == codename or repo.name == f"{codename}-updates") + # and repo.type == "deb-src" + # ) + # ]: + # return + for repo in repos: + if isinstance( + repo, (DebianRepositoryInfo, DebianDeb822RepositoryInfo) + ) and (codename in repo.name): + # split the name and components + # deb822 allows multiple suites in a single entry + # old style is one per line, so split will give either one or more + for name in repo.name.split(): + if ( + name == codename + or name == f"{codename}-updates" + and not repo.source_repo_enabled() + ): + enable_repo = f"deb-src {repo.uri} {name} {repo.components}" + node.execute( + f'echo "{enable_repo}" | sudo tee -a /etc/apt/sources.list', + shell=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + f"Could not add {repo} to /etc/apt/sources.list" + ), + ) + self.get_repositories() + + def _install_build_deps(self, packages: str) -> None: + # apt-get build-dep installs the listed build dependencies + # from the src .deb for a given package. + self._node.execute( + f"apt-get build-dep -y {packages}", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=f"Could not install apt-get build-dep for {packages}", + update_envs={"DEBIAN_FRONTEND": "noninteractive"}, + ) + def add_azure_core_repo( self, repo_name: Optional[AzureCoreRepo] = None, code_name: Optional[str] = None ) -> None: @@ -994,24 +1191,76 @@ def wait_running_package_process(self) -> None: if timeout < timer.elapsed(): raise LisaTimeoutException("timeout to wait previous dpkg process stop.") - def get_repositories(self) -> List[RepositoryInfo]: - self._initialize_package_installation() - repo_list_str = self._node.execute("apt-get update", sudo=True).stdout + def _parse_deb822_sources_file( + self, sources_file: PurePath + ) -> List[RepositoryInfo]: + repo_list_str = self._node.execute(f"cat {str(sources_file)}", sudo=True).stdout + repositories: List[RepositoryInfo] = [] + for match in _deb822_sources_entry_pattern.finditer(repo_list_str): + groups = match.groupdict() + options_long = groups.get("options", "") + # smush 'a : b' to 'a=b' + options_list = [] + enabled = True + for option_match in _deb822_option_entry_pattern.finditer(options_long): + option_matches = option_match.groupdict() + key = option_matches.get("key", "") + value = option_matches.get("value", "") + options_list += [f"{key}={value}"] + if key == "Enabled": + enabled = to_bool(value) + + repositories.append( + DebianDeb822RepositoryInfo( + type=groups.get("type", "").strip(), + uri=groups.get("uri", "").strip(), + options=" ".join(options_list), + name=groups.get("suite", "").strip(), + suite=groups.get("suite", "").strip(), + components=groups.get("components", "").strip(), + enabled=enabled, + ) + ) + return repositories + + def parse_single_line_sources_list( + self, sources_file: PurePath + ) -> List[RepositoryInfo]: + repo_list_str = self._node.execute(f"cat {str(sources_file)}", sudo=True).stdout repositories: List[RepositoryInfo] = [] for line in repo_list_str.splitlines(): matched = self._debian_repository_info_pattern.search(line) if matched: + groups = matched.groupdict() repositories.append( DebianRepositoryInfo( - name=matched.group("name"), - status=matched.group("status"), - id=matched.group("id"), - uri=matched.group("uri"), - metadata=matched.group("metadata"), + type=groups.get("type", ""), + uri=groups.get("uri", ""), + options=groups.get("options", ""), + name=groups.get( + "suite", "" + ), # need to provide name for lisa parent class + suite=groups.get("suite", ""), + components=groups.get("components", ""), ) ) + return repositories + def get_repositories(self) -> List[RepositoryInfo]: + self._initialize_package_installation() + node = self._node + repositories: List[RepositoryInfo] = list() + sources_list = node.get_pure_path("/etc/apt/sources.list") + if node.shell.exists(sources_list): + repositories += self.parse_single_line_sources_list(sources_list) + sources_fragments = node.execute( + "ls -1 /etc/apt/sources.list.d/*.sources", sudo=True, shell=True + ).stdout.splitlines() + for source_fragment in sources_fragments: + repositories += self._parse_deb822_sources_file( + node.get_pure_path(source_fragment) + ) return repositories def clean_package_cache(self) -> None: @@ -1707,6 +1956,14 @@ def _update_packages(self, packages: Optional[List[str]] = None) -> None: command += " ".join(packages) self._node.execute(command, sudo=True, timeout=3600) + def _install_build_deps(self, packages: str) -> None: + self._node.execute( + f"{self._dnf_tool()} build-dep -y {packages}", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=f"Could not install build-deps for packages, check if source repos are enabled: {packages}", + ) + class Fedora(RPMDistro): # Red Hat Enterprise Linux Server 7.8 (Maipo) => 7.8 @@ -1794,6 +2051,10 @@ def _get_information(self) -> OsInformation: return information + def _enable_package_build_deps(self) -> None: + # epel enbables source repos by default + self.install_epel() + class Redhat(Fedora): # Red Hat Enterprise Linux Server release 6.9 (Santiago) @@ -1984,7 +2245,34 @@ def name_pattern(cls) -> Pattern[str]: class AlmaLinux(Redhat): @classmethod def name_pattern(cls) -> Pattern[str]: - return re.compile("^AlmaLinux") + return re.compile("^AlmaLinux|almalinux") + + def _dnf_tool(self) -> str: + if self._node.shell.exists(self._node.get_pure_path("/usr/bin/dnf")): + return "dnf" + else: + return "yum" + + def _enable_package_build_deps(self) -> None: + # enable epel first + super()._enable_package_build_deps() + # almalinux has a few different source repos, they are easy to enable + self._node.execute("dnf install -y almalinux-release-devel") + # then enable crb (code ready builder) using alma tool + # this enables some needed package build dependency srpms + # this was formerly known as the 'powertools' repo. + if int(self.information.version.major) == 8: + pkg = "powertools" + elif int(self.information.version.major) == 9: + pkg = "crb" + # set the repo as enabled + self._node.execute( + f"dnf config-manager --set-enabled {pkg}", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=f"Could not enable source build repo/pkg: {pkg}", + ) + self._node.execute("/usr/bin/crb enable", sudo=True) class CBLMariner(RPMDistro): @@ -2296,6 +2584,45 @@ def add_azure_core_repo( repo_name="packages-microsoft-com-azurecore", ) + def _enable_package_build_deps(self) -> None: + # zypper seems more suited to interactive use + # to enable source repos, list the defaults and select the source repos + repos = self.get_repositories() + for repo in repos: + if isinstance(repo, SuseRepositoryInfo) and "Source-Pool" in repo.name: + # if it's a source repo, enable it + self._node.execute( + f"zypper mr -e {repo.alias}", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + f"Could not enable source pool for repo: {repo.alias}" + ), + ) + # then refresh the repo status to fetch the new metadata + self._node.execute( + "zypper refresh -y", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "Failure to zypper refresh after enabling source repos." + ), + ) + + def _install_build_deps(self, packages: str) -> None: + # zypper sourceinstall the packages + # if there are missing dependencies, you can attempt to + # force zypper to work out the problem. It seems to assume + # interactive usage for these even with the -n flag so YMMV. + self._node.execute( + f"zypper si --build-deps-only --force-resolution {packages}", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + f"failed to source install package: {packages}" + ), + ) + def _initialize_package_installation(self) -> None: self.wait_running_process("zypper") service = self._node.tools[Service] diff --git a/microsoft/testsuites/dpdk/common.py b/microsoft/testsuites/dpdk/common.py index 6afb326e0c..bf64bb1eea 100644 --- a/microsoft/testsuites/dpdk/common.py +++ b/microsoft/testsuites/dpdk/common.py @@ -12,7 +12,7 @@ from lisa import Node from lisa.executable import Tool from lisa.operating_system import Debian, Fedora, Oracle, Posix, Suse, Ubuntu -from lisa.tools import Git, Lscpu, Tar, Wget +from lisa.tools import Git, Lscpu, Lsmod, Tar, Wget from lisa.tools.lscpu import CpuArchitecture from lisa.util import UnsupportedDistroException @@ -39,10 +39,12 @@ def __init__( matcher: Callable[[Posix], bool], packages: Optional[Sequence[Union[str, Tool, Type[Tool]]]] = None, stop_on_match: bool = False, + build_deps: bool = False, ) -> None: self.matcher = matcher self.packages = packages self.stop_on_match = stop_on_match + self.build_deps = build_deps class DependencyInstaller: @@ -62,12 +64,19 @@ def install_required_packages( # find the match for an OS, install the packages. # stop on list end or if exclusive_match parameter is true. packages: List[Union[str, Tool, Type[Tool]]] = [] + build_deps: List[Union[str, Tool, Type[Tool]]] = [] for requirement in self.requirements: if requirement.matcher(os) and requirement.packages: - packages += requirement.packages + if requirement.build_deps: + build_deps += requirement.packages + else: + packages += requirement.packages if requirement.stop_on_match: break - os.install_packages(packages=packages, extra_args=extra_args) + if build_deps: + os.install_build_deps(build_deps) + if packages: + os.install_packages(packages=packages, extra_args=extra_args) # NOTE: It is up to the caller to raise an exception on an invalid OS @@ -317,19 +326,21 @@ def check_dpdk_support(node: Node) -> None: raise UnsupportedDistroException( node.os, "ARM64 tests are only supported on Ubuntu + failsafe." ) - if isinstance(node.os, Debian): - if isinstance(node.os, Ubuntu): - node.log.debug( - "Checking Ubuntu release: " - f"is_latest_or_prerelease? ({is_ubuntu_latest_or_prerelease(node.os)})" - f" is_lts_version? ({is_ubuntu_lts_version(node.os)})" - ) - # TODO: undo special casing for 18.04 when it's usage is less common - supported = ( - node.os.information.version == "18.4.0" - or is_ubuntu_latest_or_prerelease(node.os) - or is_ubuntu_lts_version(node.os) - ) + if isinstance(node.os, Ubuntu): + node.log.debug( + "Checking Ubuntu release: " + f"is_latest_or_prerelease? ({is_ubuntu_latest_or_prerelease(node.os)})" + f" is_lts_version? ({is_ubuntu_lts_version(node.os)})" + ) + # TODO: undo special casing for 18.04 when it's usage is less common + supported = ( + node.os.information.version == "18.4.0" + or is_ubuntu_latest_or_prerelease(node.os) + or is_ubuntu_lts_version(node.os) + ) + elif isinstance(node.os, Debian): + if node.nics.is_mana_device_present(): + supported = node.os.information.version >= "13.0.0" else: supported = node.os.information.version >= "11.0.0" elif isinstance(node.os, Fedora) and not isinstance(node.os, Oracle): @@ -346,13 +357,18 @@ def check_dpdk_support(node: Node) -> None: isinstance(node.os, (Debian, Fedora, Suse, Fedora)) and node.nics.is_mana_device_present() ): - # NOTE: Kernel backport examples are available for lower kernels. - # HOWEVER: these are not suitable for general testing and should be installed - # in the image _before_ starting the test. - # ex: make a SIG image first using the kernel build transformer. - if node.os.get_kernel_information().version < "5.15.0": + # don't assume kernel version, check for the drivers. + # This modprobe call is truly a "don't care". + # It will load mana_ib if it's present so this next check can run. + node.execute("modprobe mana_ib", sudo=True) + if not ( + node.nics.is_mana_driver_enabled() + # risk of driver not being installed until after an update here + and node.nics.is_mana_ib_driver_enabled() + ): raise UnsupportedDistroException( - node.os, "MANA driver is not available for kernel < 5.15" + node.os, + "mana/mana_ib driver is not available on this image.", ) if not supported: raise UnsupportedDistroException( diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 468830f3c8..d8947aff5a 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -123,16 +123,12 @@ ), OsPackageDependencies( matcher=lambda x: isinstance(x, Debian), - packages=[ - "build-essential", - "libnuma-dev", - "libmnl-dev", - "python3-pyelftools", - "libelf-dev", - "pkg-config", - ], + packages=["dpdk", "dpdk-dev"], stop_on_match=True, + build_deps=True, ), + # todo: suse build-dep is tricky for noninteractive + # so just use the old 'list of package names' for dependencies OsPackageDependencies( matcher=lambda x: isinstance(x, Suse), packages=[ @@ -145,18 +141,19 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, (Fedora)), - packages=[ - "psmisc", - "numactl-devel", - "pkgconfig", - "elfutils-libelf-devel", - "python3-pip", - "kernel-modules-extra", - "kernel-headers", - "gcc-c++", - ], + # alma/rocky have started + # including testpmd by default in 'dpdk' + matcher=lambda x: isinstance(x, Fedora) + and not x.is_package_in_repo("dpdk-devel"), + packages=["dpdk"], + stop_on_match=True, + build_deps=True, + ), + OsPackageDependencies( + matcher=lambda x: isinstance(x, (Fedora, Suse)), + packages=["dpdk", "dpdk-devel"], stop_on_match=True, + build_deps=True, ), OsPackageDependencies(matcher=unsupported_os_thrower), ] @@ -225,8 +222,7 @@ def _setup_node(self) -> None: # install( Tool ) doesn't seem to install the tool until it's used :\ # which breaks when another tool checks for it's existence before building... # like cmake, meson, make, autoconf, etc. - self._node.tools[Ninja].install() - self._node.tools[Pip].install_packages("pyelftools") + self._os.install_build_deps("dpdk") def _uninstall(self) -> None: # undo source installation (thanks ninja) @@ -798,7 +794,9 @@ def _install(self) -> bool: # bionic needs to update to latest first node.os.update_packages("") if self.is_mana and not ( - isinstance(node.os, Ubuntu) or isinstance(node.os, Fedora) + isinstance(node.os, Ubuntu) + or isinstance(node.os, Fedora) + or (isinstance(node.os, Debian) and node.os.information.version >= "13.0.0") ): raise SkippedException("MANA DPDK test is not supported on this OS") diff --git a/microsoft/testsuites/dpdk/rdmacore.py b/microsoft/testsuites/dpdk/rdmacore.py index 0a0d650e1b..519fbf347e 100644 --- a/microsoft/testsuites/dpdk/rdmacore.py +++ b/microsoft/testsuites/dpdk/rdmacore.py @@ -28,53 +28,10 @@ packages=["linux-modules-extra-azure"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian), - packages=[ - "cmake", - "libudev-dev", - "libnl-3-dev", - "libnl-route-3-dev", - "ninja-build", - "pkg-config", - "valgrind", - "python3-dev", - "cython3", - "python3-docutils", - "pandoc", - "libssl-dev", - "libelf-dev", - "python3-pip", - "libnuma-dev", - ], - stop_on_match=True, - ), - OsPackageDependencies( - matcher=lambda x: isinstance(x, Fedora), - packages=[ - "cmake", - "libudev-devel", - "libnl3-devel", - "pkg-config", - "valgrind", - "python3-devel", - "openssl-devel", - "unzip", - "elfutils-devel", - "python3-pip", - "tar", - "wget", - "dos2unix", - "psmisc", - "kernel-devel-$(uname -r)", - "librdmacm-devel", - "libmnl-devel", - "kernel-modules-extra", - "numactl-devel", - "kernel-headers", - "elfutils-libelf-devel", - "libbpf-devel", - ], + matcher=lambda x: isinstance(x, (Fedora, Debian)), + packages=["rdma-core"], stop_on_match=True, + build_deps=True, ), # FIXME: SUSE rdma-core build packages not implemented # for source builds.