diff --git a/Makefile.work b/Makefile.work index 11f9e50b4f4d..71e83e9eb1b8 100644 --- a/Makefile.work +++ b/Makefile.work @@ -198,6 +198,7 @@ $(shell \ SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \ DBGOPT='$(DBGOPT)' \ + MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \ scripts/generate_buildinfo_config.sh) # Generate the slave Dockerfile, and prepare build info for it @@ -533,6 +534,7 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \ MIRROR_URLS=$(MIRROR_URLS) \ MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \ GZ_COMPRESS_PROGRAM=$(GZ_COMPRESS_PROGRAM) \ + MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \ $(SONIC_OVERRIDE_BUILD_VARS) .PHONY: sonic-slave-build sonic-slave-bash init reset diff --git a/rules/config b/rules/config index d37699433dcf..c01adc0f5390 100644 --- a/rules/config +++ b/rules/config @@ -234,6 +234,9 @@ TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,htt # docker: docker base images SONIC_VERSION_CONTROL_COMPONENTS ?= none +# MIRROR_SNAPSHOT - support mirror snapshot flag +MIRROR_SNAPSHOT ?= n + # SONIC_VERSION_CACHE allows the .deb,.py, wget, git, docker and go files to be stored in the cache path. This allows the submodules to # cache standard installation package and restored back to avoid the package download every time. # SONIC_VERSION_CACHE - Method of deb package caching diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh index 5f94e701cbde..a1e5900a48b8 100755 --- a/scripts/build_mirror_config.sh +++ b/scripts/build_mirror_config.sh @@ -5,16 +5,40 @@ CONFIG_PATH=$1 export ARCHITECTURE=$2 export DISTRIBUTION=$3 +DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net +MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror +[ -f target/versions/default/versions-mirror ] && MIRROR_VERSION_FILE=target/versions/default/versions-mirror + # The default mirror urls DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/ DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/ + # The debian-archive.trafficmanager.net does not support armhf, use debian.org instead if [ "$ARCHITECTURE" == "armhf" ]; then DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/ DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/ fi +if [ "$MIRROR_SNAPSHOT" == y ]; then + if [ -f $MIRROR_VERSION_FILE ]; then + DEBIAN_TIMESTAMP=$(grep "^debian==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//') + DEBIAN_SECURITY_TIMESTAMP=$(grep "^debian-security==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//') + elif [ -z "$DEBIAN_TIMESTAMP" ] || [ -z "$DEBIAN_SECURITY_TIMESTAMP" ]; then + DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp) + DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp) + fi + + DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/snapshot/debian/$DEBIAN_TIMESTAMP/ + DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/snapshot/debian-security/$DEBIAN_SECURITY_TIMESTAMP/ + + mkdir -p target/versions/default + if [ ! -f target/versions/default/versions-mirror ]; then + echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror + echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror + fi +fi + [ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS [ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS @@ -24,3 +48,7 @@ TEMPLATE=files/apt/sources.list.j2 [ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2 MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE +if [ "$MIRROR_SNAPSHOT" == y ]; then + # Set the snapshot mirror, and add the SET_REPR_MIRRORS flag + sed -i -e "/^#*deb.*packages.trafficmanager.net/! s/^#*deb/#&/" -e "\$a#SET_REPR_MIRRORS" $CONFIG_PATH/sources.list.$ARCHITECTURE +fi diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh index b0ec54924246..87eb612007ba 100755 --- a/scripts/generate_buildinfo_config.sh +++ b/scripts/generate_buildinfo_config.sh @@ -11,3 +11,4 @@ echo "export SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" echo "export SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}" >> $BUILDINFO_CONFIG echo "export SONIC_VERSION_CACHE_SOURCE=${SONIC_VERSION_CACHE_SOURCE}" >> $BUILDINFO_CONFIG echo "export DISTRO=${DISTRO}" >> $BUILDINFO_CONFIG +echo "export MIRROR_SNAPSHOT=$MIRROR_SNAPSHOT" >> $BUILDINFO_CONFIG diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py index 27b757721f25..50fd9452454b 100755 --- a/scripts/versions_manager.py +++ b/scripts/versions_manager.py @@ -562,12 +562,13 @@ def filter(self, ctypes=[]): module.filter(ctypes=ctypes) def get_default_module(self): - if DEFAULT_MODULE in self.modules: - return self.modules[DEFAULT_MODULE] + default_module = self.modules.get(DEFAULT_MODULE, VersionModule(DEFAULT_MODULE, [])) ctypes = self.get_component_types() dists = self.get_dists() components = [] for ctype in ctypes: + if ctype in DEFAULT_OVERWRITE_COMPONENTS: + continue if ctype == 'deb': for dist in dists: versions = self._get_versions(ctype, dist) @@ -579,7 +580,9 @@ def get_default_module(self): common_versions = self._get_common_versions(versions) component = Component(self.verbose, common_versions, ctype) components.append(component) - return VersionModule(self.verbose, DEFAULT_MODULE, components) + module = VersionModule(self.verbose, DEFAULT_MODULE, components) + module.overwrite(default_module, True, True) + return module def get_aggregatable_modules(self): modules = {} diff --git a/slave.mk b/slave.mk index da22f7fb1bc1..2d8006fb8aad 100644 --- a/slave.mk +++ b/slave.mk @@ -87,6 +87,7 @@ export CROSS_BUILD_ENVIRON export BLDENV export BUILD_WORKDIR export GZ_COMPRESS_PROGRAM +export MIRROR_SNAPSHOT ############################################################################### ## Utility rules diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh index 38d2e8881588..b42e37bf62b8 100755 --- a/src/sonic-build-hooks/scripts/buildinfo_base.sh +++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh @@ -11,7 +11,7 @@ POST_VERSION_PATH=$BUILDINFO_PATH/post-versions VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb WEB_VERSION_FILE=$VERSION_PATH/versions-web BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web -REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/debian' +REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/' DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock . $BUILDINFO_PATH/config/buildinfo.config @@ -112,14 +112,39 @@ set_reproducible_mirrors() { # Remove the charater # in front of the line if matched local expression="s/^#\(.*$REPR_MIRROR_URL_PATTERN\)/\1/" + # Add the character # in front of the line, if not match the URL pattern condition + local expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#*deb/#&/" + local expression3="\$a#SET_REPR_MIRRORS" if [ "$1" = "-d" ]; then # Add the charater # in front of the line if match expression="s/^deb.*$REPR_MIRROR_URL_PATTERN/#\0/" + # Remove the character # in front of the line, if not match the URL pattern condition + expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#(#*deb)/\1/" + expression3="/#SET_REPR_MIRRORS/d" fi local mirrors="/etc/apt/sources.list $(find /etc/apt/sources.list.d/ -type f)" for mirror in $mirrors; do + if ! grep -iq "$REPR_MIRROR_URL_PATTERN" "$mirror"; then + continue + fi + + # Make sure no duplicate operations on the mirror config file + if ([ "$1" == "-d" ] && ! grep -iq "#SET_REPR_MIRRORS" "$mirror") || + ([ "$1" != "-d" ] && grep -iq "#SET_REPR_MIRRORS" "$mirror"); then + continue + fi + + # Enable or disable the reproducible mirrors $SUDO sed -i "$expression" "$mirror" + + # Enable or disable the none reproducible mirrors + if [ "$MIRROR_SNAPSHOT" == y ]; then + $SUDO sed -ri "$expression2" "$mirror" + fi + + # Add or remove the SET_REPR_MIRRORS flag + $SUDO sed -i "$expression3" "$mirror" done } diff --git a/src/sonic-build-hooks/scripts/collect_version_files b/src/sonic-build-hooks/scripts/collect_version_files index 6e082406da56..081ded342c4d 100755 --- a/src/sonic-build-hooks/scripts/collect_version_files +++ b/src/sonic-build-hooks/scripts/collect_version_files @@ -25,6 +25,13 @@ dpkg-query -W -f '${Package}==${Version}\n' | grep -Ev "${SKIP_VERSION_PACKAGE}" ## Add the the packages purged [ -f $POST_VERSION_PATH/purge-versions-deb ] && cat $POST_VERSION_PATH/purge-versions-deb >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" +## Add mirror versions +while read -r line; do + mirror=$(echo "$line" | sed "s/.*\///" | sed "s/_InRelease.*//") + date=$(date --date="$(echo "$line" | cut -d: -f3-)" +%Y-%m-%dT%H:%M:%SZ) + echo "$mirror==$date" >> ${TARGET_PATH}/versions-mirror +done < <(grep Date: /var/lib/apt/lists/*_InRelease 2>/dev/null) + ## Print the unique and sorted result sort -u "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" if [ -e "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" ]; then @@ -33,5 +40,8 @@ fi if [ -e "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" ]; then sort -u "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" fi +if [ -e "${TARGET_PATH}/versions-mirror" ]; then + sort -u "${TARGET_PATH}/versions-mirror" -o "${TARGET_PATH}/versions-mirror" +fi exit 0