diff --git a/hpccm/building_blocks/apt_get.py b/hpccm/building_blocks/apt_get.py index 1e2fa0a..a1664db 100644 --- a/hpccm/building_blocks/apt_get.py +++ b/hpccm/building_blocks/apt_get.py @@ -83,7 +83,7 @@ def __init__(self, **kwargs): super(apt_get, self).__init__() - self.__apt_key = kwargs.get('_apt_key', True) + self.__apt_key = kwargs.get('_apt_key', False) self.__aptitude = kwargs.get('aptitude', False) self.__commands = [] self.__download = kwargs.get('download', False) diff --git a/hpccm/building_blocks/intel_mpi.py b/hpccm/building_blocks/intel_mpi.py index d95c173..983893c 100644 --- a/hpccm/building_blocks/intel_mpi.py +++ b/hpccm/building_blocks/intel_mpi.py @@ -126,6 +126,7 @@ def __instructions(self): raise RuntimeError('Intel EULA was not accepted. To accept, see the documentation for this building block') self += packages( + _apt_key=True, apt_keys=['https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-{}.PUB'.format(self.__year)], apt_repositories=['deb https://apt.repos.intel.com/mpi all main'], ospackages=['intel-mpi-{}'.format(self.__version)], diff --git a/hpccm/building_blocks/intel_psxe_runtime.py b/hpccm/building_blocks/intel_psxe_runtime.py index 96c0091..371e7ad 100644 --- a/hpccm/building_blocks/intel_psxe_runtime.py +++ b/hpccm/building_blocks/intel_psxe_runtime.py @@ -176,6 +176,7 @@ def __instructions(self): self += packages( apt=self.__apt, + _apt_key=True, apt_keys = ['https://apt.repos.intel.com/{0}/GPG-PUB-KEY-INTEL-PSXE-RUNTIME-{0}'.format(self.__year)], apt_repositories=apt_repositories, aptitude=True, diff --git a/hpccm/building_blocks/llvm.py b/hpccm/building_blocks/llvm.py index d276cef..9e4d922 100644 --- a/hpccm/building_blocks/llvm.py +++ b/hpccm/building_blocks/llvm.py @@ -381,8 +381,8 @@ def __upstream_package_repos(self): raise RuntimeError('Unsupported Ubuntu version') return [ - 'deb http://apt.llvm.org/{0}/ llvm-toolchain-{1} main'.format(codename, codename_ver), - 'deb-src http://apt.llvm.org/{0}/ llvm-toolchain-{1} main'.format(codename, codename_ver)] + 'deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/{0}/ llvm-toolchain-{1} main'.format(codename, codename_ver), + 'deb-src [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/{0}/ llvm-toolchain-{1} main'.format(codename, codename_ver)] def runtime(self, _from='0'): """Generate the set of instructions to install the runtime specific diff --git a/hpccm/building_blocks/mkl.py b/hpccm/building_blocks/mkl.py index 0d7893d..dfbd39c 100644 --- a/hpccm/building_blocks/mkl.py +++ b/hpccm/building_blocks/mkl.py @@ -118,6 +118,7 @@ def __instructions(self): raise RuntimeError('Intel EULA was not accepted. To accept, see the documentation for this building block') self += packages( + _apt_key=True, apt_keys=['https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-{}.PUB'.format(self.__year)], apt_repositories=['deb https://apt.repos.intel.com/mkl all main'], ospackages=['intel-mkl-64bit-{}'.format(self.__version)], diff --git a/hpccm/building_blocks/mlnx_ofed.py b/hpccm/building_blocks/mlnx_ofed.py index 8f2f170..993a43f 100644 --- a/hpccm/building_blocks/mlnx_ofed.py +++ b/hpccm/building_blocks/mlnx_ofed.py @@ -133,6 +133,7 @@ def __instructions(self): self += packages(ospackages=self.__ospackages) self += packages( + _apt_key=True, apt_keys=[self.__key], apt_repositories=['https://linux.mellanox.com/public/repo/mlnx_ofed/{0}/{1}/mellanox_mlnx_ofed.list'.format(self.__version, self.__oslabel)], download=bool(self.__prefix), diff --git a/hpccm/building_blocks/nccl.py b/hpccm/building_blocks/nccl.py index be86250..64bb210 100644 --- a/hpccm/building_blocks/nccl.py +++ b/hpccm/building_blocks/nccl.py @@ -124,7 +124,7 @@ def __init__(self, **kwargs): 'libnccl-dev={0}+cuda{1}'.format(self.__version, self.__cuda)], apt_keys=['https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1}/3bf863cc.pub'.format(self.__distro_label, get_cpu_architecture())], - apt_repositories=['deb https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1} /'.format(self.__distro_label, get_cpu_architecture())], + apt_repositories=['deb [signed-by=/usr/share/keyrings/3bf863cc.gpg] https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1} /'.format(self.__distro_label, get_cpu_architecture())], yum=['libnccl-{0}+cuda{1}'.format(self.__version, self.__cuda), 'libnccl-devel-{0}+cuda{1}'.format(self.__version, self.__cuda)], @@ -247,7 +247,7 @@ def runtime(self, _from='0'): apt=['libnccl2={0}+cuda{1}'.format(self.__version, self.__cuda)], apt_keys=['https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1}/3bf863cc.pub'.format(self.__distro_label, get_cpu_architecture())], - apt_repositories=['deb https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1} /'.format(self.__distro_label, get_cpu_architecture())], + apt_repositories=['deb [signed-by=/usr/share/keyrings/3bf863cc.gpg] https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1} /'.format(self.__distro_label, get_cpu_architecture())], yum=['libnccl-{0}+cuda{1}'.format(self.__version, self.__cuda)], yum_keys=['https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1}/3bf863cc.pub'.format(self.__distro_label, get_cpu_architecture())], yum_repositories=['https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1}'.format(self.__distro_label, get_cpu_architecture())]) diff --git a/hpccm/building_blocks/packages.py b/hpccm/building_blocks/packages.py index ae8b7aa..b67ef87 100644 --- a/hpccm/building_blocks/packages.py +++ b/hpccm/building_blocks/packages.py @@ -134,7 +134,7 @@ def __init__(self, **kwargs): super(packages, self).__init__() self.__apt = kwargs.get('apt', []) - self.__apt_key = kwargs.get('_apt_key', True) + self.__apt_key = kwargs.get('_apt_key', False) self.__apt_keys = kwargs.get('apt_keys', []) self.__apt_ppas = kwargs.get('apt_ppas', []) self.__apt_repositories = kwargs.get('apt_repositories', []) diff --git a/test/test_apt_get.py b/test/test_apt_get.py index 639a82c..2ec68c1 100644 --- a/test/test_apt_get.py +++ b/test/test_apt_get.py @@ -44,27 +44,11 @@ def test_basic(self): gfortran && \ rm -rf /var/lib/apt/lists/*''') - @ubuntu - @docker - def test_add_repo(self): - """Add repo and key""" - a = apt_get(keys=['https://www.example.com/key.pub'], - ospackages=['example'], - repositories=['deb http://www.example.com all main']) - self.assertEqual(str(a), -r'''RUN wget -qO - https://www.example.com/key.pub | apt-key add - && \ - echo "deb http://www.example.com all main" >> /etc/apt/sources.list.d/hpccm.list && \ - apt-get update -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - example && \ - rm -rf /var/lib/apt/lists/*''') - @ubuntu @docker def test_add_repo_signed_by(self): """Add repo and key, using the signed-by method rather than apt-key""" - a = apt_get(_apt_key=False, - keys=['https://www.example.com/key.pub'], + a = apt_get(keys=['https://www.example.com/key.pub'], ospackages=['example'], repositories=['deb [signed-by=/usr/share/keyrings/key.gpg] http://www.example.com all main']) self.assertEqual(str(a), diff --git a/test/test_llvm.py b/test/test_llvm.py index 0cccd65..3e35935 100644 --- a/test/test_llvm.py +++ b/test/test_llvm.py @@ -330,9 +330,11 @@ def test_upstream_ubuntu16(self): gnupg \ wget && \ rm -rf /var/lib/apt/lists/* -RUN wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" >> /etc/apt/sources.list.d/hpccm.list && \ - echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" >> /etc/apt/sources.list.d/hpccm.list && \ +RUN mkdir -p /usr/share/keyrings && \ + rm -f /usr/share/keyrings/llvm-snapshot.gpg.gpg && \ + wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /usr/share/keyrings/llvm-snapshot.gpg.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" >> /etc/apt/sources.list.d/hpccm.list && \ + echo "deb-src [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" >> /etc/apt/sources.list.d/hpccm.list && \ apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ clang-10 \ @@ -356,9 +358,11 @@ def test_upstream_ubuntu18(self): gnupg \ wget && \ rm -rf /var/lib/apt/lists/* -RUN wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/sources.list.d/hpccm.list && \ - echo "deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/sources.list.d/hpccm.list && \ +RUN mkdir -p /usr/share/keyrings && \ + rm -f /usr/share/keyrings/llvm-snapshot.gpg.gpg && \ + wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /usr/share/keyrings/llvm-snapshot.gpg.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/sources.list.d/hpccm.list && \ + echo "deb-src [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/sources.list.d/hpccm.list && \ apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ clang-18 \ @@ -386,9 +390,11 @@ def test_upstream_ubuntu24(self): gnupg \ wget && \ rm -rf /var/lib/apt/lists/* -RUN wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble main" >> /etc/apt/sources.list.d/hpccm.list && \ - echo "deb-src http://apt.llvm.org/noble/ llvm-toolchain-noble main" >> /etc/apt/sources.list.d/hpccm.list && \ +RUN mkdir -p /usr/share/keyrings && \ + rm -f /usr/share/keyrings/llvm-snapshot.gpg.gpg && \ + wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /usr/share/keyrings/llvm-snapshot.gpg.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/noble/ llvm-toolchain-noble main" >> /etc/apt/sources.list.d/hpccm.list && \ + echo "deb-src [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/noble/ llvm-toolchain-noble main" >> /etc/apt/sources.list.d/hpccm.list && \ apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ clang-18 \ @@ -415,9 +421,11 @@ def test_upstream_ubuntu20(self): gnupg \ wget && \ rm -rf /var/lib/apt/lists/* -RUN wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal main" >> /etc/apt/sources.list.d/hpccm.list && \ - echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal main" >> /etc/apt/sources.list.d/hpccm.list && \ +RUN mkdir -p /usr/share/keyrings && \ + rm -f /usr/share/keyrings/llvm-snapshot.gpg.gpg && \ + wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /usr/share/keyrings/llvm-snapshot.gpg.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/focal/ llvm-toolchain-focal main" >> /etc/apt/sources.list.d/hpccm.list && \ + echo "deb-src [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/focal/ llvm-toolchain-focal main" >> /etc/apt/sources.list.d/hpccm.list && \ apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ clang-18 \ @@ -445,9 +453,11 @@ def test_upstream_aarch64(self): gnupg \ wget && \ rm -rf /var/lib/apt/lists/* -RUN wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-11 main" >> /etc/apt/sources.list.d/hpccm.list && \ - echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-11 main" >> /etc/apt/sources.list.d/hpccm.list && \ +RUN mkdir -p /usr/share/keyrings && \ + rm -f /usr/share/keyrings/llvm-snapshot.gpg.gpg && \ + wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /usr/share/keyrings/llvm-snapshot.gpg.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/xenial/ llvm-toolchain-xenial-11 main" >> /etc/apt/sources.list.d/hpccm.list && \ + echo "deb-src [signed-by=/usr/share/keyrings/llvm-snapshot.gpg.gpg] http://apt.llvm.org/xenial/ llvm-toolchain-xenial-11 main" >> /etc/apt/sources.list.d/hpccm.list && \ apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ clang-11 \ diff --git a/test/test_nccl.py b/test/test_nccl.py index df23c72..da2ee91 100644 --- a/test/test_nccl.py +++ b/test/test_nccl.py @@ -46,8 +46,10 @@ def test_defaults_ubuntu(self): gnupg \ wget && \ rm -rf /var/lib/apt/lists/* -RUN wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub | apt-key add - && \ - echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \ +RUN mkdir -p /usr/share/keyrings && \ + rm -f /usr/share/keyrings/3bf863cc.gpg && \ + wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub | gpg --dearmor -o /usr/share/keyrings/3bf863cc.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/3bf863cc.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \ apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libnccl-dev=2.12.10-1+cuda11.6 \ @@ -69,8 +71,10 @@ def test_defaults_ubuntu18(self): gnupg \ wget && \ rm -rf /var/lib/apt/lists/* -RUN wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub | apt-key add - && \ - echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \ +RUN mkdir -p /usr/share/keyrings && \ + rm -f /usr/share/keyrings/3bf863cc.gpg && \ + wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub | gpg --dearmor -o /usr/share/keyrings/3bf863cc.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/3bf863cc.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \ apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libnccl-dev=2.12.10-1+cuda11.6 \ @@ -92,8 +96,10 @@ def test_ubuntu_ppc64le(self): gnupg \ wget && \ rm -rf /var/lib/apt/lists/* -RUN wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/ppc64el/3bf863cc.pub | apt-key add - && \ - echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/ppc64el /" >> /etc/apt/sources.list.d/hpccm.list && \ +RUN mkdir -p /usr/share/keyrings && \ + rm -f /usr/share/keyrings/3bf863cc.gpg && \ + wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/ppc64el/3bf863cc.pub | gpg --dearmor -o /usr/share/keyrings/3bf863cc.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/3bf863cc.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/ppc64el /" >> /etc/apt/sources.list.d/hpccm.list && \ apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libnccl-dev=2.4.8-1+cuda9.2 \ @@ -217,8 +223,10 @@ def test_runtime(self): gnupg \ wget && \ rm -rf /var/lib/apt/lists/* -RUN wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub | apt-key add - && \ - echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \ +RUN mkdir -p /usr/share/keyrings && \ + rm -f /usr/share/keyrings/3bf863cc.gpg && \ + wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub | gpg --dearmor -o /usr/share/keyrings/3bf863cc.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/3bf863cc.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \ apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libnccl2=2.12.10-1+cuda11.6 && \