fetch pseudo with https protocol

bitbake/lib/bb/runqueue.py

@@ -2443,6 +2443,11 @@ def process_possible_migrations(self):
 
         if update_tasks:
             self.sqdone = False
+            for tid in [t[0] for t in update_tasks]:
+                h = pending_hash_index(tid, self.rqdata)
+                if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]:
+                    self.sq_deferred[tid] = self.sqdata.hashes[h]
+                    bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h]))
             update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
 
         for (tid, harddepfail, origvalid) in update_tasks:
@@ -2786,6 +2791,19 @@ def process_endpoints(endpoints):
     sqdata.stamppresent = set()
     sqdata.valid = set()
 
+    sqdata.hashes = {}
+    sqrq.sq_deferred = {}
+    for mc in sorted(sqdata.multiconfigs):
+        for tid in sorted(sqdata.sq_revdeps):
+            if mc_from_tid(tid) != mc:
+                continue
+            h = pending_hash_index(tid, rqdata)
+            if h not in sqdata.hashes:
+                sqdata.hashes[h] = tid
+            else:
+                sqrq.sq_deferred[tid] = sqdata.hashes[h]
+                bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h]))
+
     update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True)
 
     # Compute a list of 'stale' sstate tasks where the current hash does not match the one
@@ -2850,32 +2868,20 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
 
     sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary)
 
-    sqdata.hashes = {}
-    sqrq.sq_deferred = {}
-    for mc in sorted(sqdata.multiconfigs):
-        for tid in sorted(sqdata.sq_revdeps):
-            if mc_from_tid(tid) != mc:
-                continue
-            if tid in sqdata.stamppresent:
-                continue
-            if tid in sqdata.valid:
-                continue
-            if tid in sqdata.noexec:
-                continue
-            if tid in sqrq.scenequeue_notcovered:
-                continue
-            if tid in sqrq.scenequeue_covered:
-                continue
-
-            h = pending_hash_index(tid, rqdata)
-            if h not in sqdata.hashes:
-                if tid in tids:
-                    sqdata.outrightfail.add(tid)
-                sqdata.hashes[h] = tid
-            else:
-                sqrq.sq_deferred[tid] = sqdata.hashes[h]
-                bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h]))
-
+    for tid in tids:
+        if tid in sqdata.stamppresent:
+            continue
+        if tid in sqdata.valid:
+            continue
+        if tid in sqdata.noexec:
+            continue
+        if tid in sqrq.scenequeue_covered:
+            continue
+        if tid in sqrq.scenequeue_notcovered:
+            continue
+        if tid in sqrq.sq_deferred:
+            continue
+        sqdata.outrightfail.add(tid)
 
 class TaskFailure(Exception):
     """

documentation/test-manual/index.rst

@@ -13,6 +13,8 @@ Yocto Project Test Environment Manual
    intro
    test-process
    understand-autobuilder
+   reproducible-builds
+   yocto-project-compatible
    history
 
 .. include:: /boilerplate.rst

documentation/test-manual/reproducible-builds.rst

@@ -0,0 +1,56 @@
+.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
+
+*******************
+Reproducible Builds
+*******************
+
+================
+How we define it
+================
+
+The Yocto Project defines reproducibility as being where a given input build configuration will give the same binary output regardless of when it is built (now or in 5 years time), regardless of the path on the filesystem the build is run in and regardless of the distro and tools on the underlying host system the build is running on.
+
+==============
+Why it matters
+==============
+
+The project aligns with the Reproducibile Builds project (https://reproducible-builds.org/) and they have information about why it matters. In addition to being able to validate for security issues being introduced which they talk about at length, from a Yocto Project perspective, it is hugely important that our builds are deterministic. We expect that when you build a given input set of metadata, you get consistent output. This has always been a key focus but is now true down to the binary level including timestamps.
+
+For example, if you find at some point in the future life of a product that you need to rebuild to add a security fix, only the components that have changed should change at the binary level leading to much easier and clearer bounds on where validation is needed.
+
+This also gives an additional benefit to the project builds themselves, our hash equivalence for sstate object reuse works much more effecitvely when the binary outputis unchanging.
+
+===================
+How we implement it
+===================
+
+We add mappings to the compiler options to ensure debug filepaths are mapped to consistent target compatible paths
+
+We are explict about recipe dependencies and their configuration (no floating configure options or host dependencies creeping in)
+
+We have recipe specific sysroots to isolate recipes so they only see their dependencies.
+
+We filter the tools availble from the host's PATH through HOSTTOOLS.
+
+=========================================
+Can we prove the project is reproducible?
+=========================================
+
+Yes, we can prove it and we now regularlly test this on the autobuilder. At the time of writing, OE-Core is 100% reproducible for all it's recipes (i.e. world builds) apart from go-lang and ruby's docs package. go-lang has some fundamental problems with reproducibility as it currently always depends upon the paths it is build in unfortunately.
+
+[Info about what we run]
+
+[Add info about diffoscope]
+
+You can see the current status at: https://www.yoctoproject.org/reproducible-build-results/
+
+====================
+Can I test my layer?
+====================
+
+[Yes, add instructions]
+
+
+
+
+

documentation/test-manual/yocto-project-compatible.rst

@@ -0,0 +1,52 @@
+.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
+
+************************
+Yocto Project Compatible
+************************
+
+============
+Introduction
+============
+
+After the introduction of layers to OpenEmbedded, it quickly became clear that some layers were popular and worked well, others developed a reputation for being 'problematic'. Those were layers which didn't inter-operate well with other layers and tended to assume they controlled all the aspects of the end resulting output. This isn't usually intentional but because they're often developed by developers with a particular focus (e.g. a company's BSP) whilst the end users have a different focus (e.g. integrating that BSP into a product).
+
+As a result of noticing patterns like this and friction between layers, the project developed the "Yocto Project Compatible" badge programme where layers modelling the best known practises could be marked as being widely compatible with other layers. This takes the form of a set of yes/no binary answer questions where layers can declare if they have met the appropriate criteria. In the second version of the programme, a script was added to make validation easier and clearer, the script is called 'yocto-check-layer' and is available in OpenEmbedded-Core.
+
+========
+Benefits
+========
+
+The OpenEmbedded Layer model is powerful and flexible, it gives users the ultimate power to change pretty much any aspect of the system but as with most things, power comes with responsibility. The Yocto Project would like to see people able to mix and match BSPs with distro configs or software stacks and be able to merge these together such that the result inter-operates well together. Over time, the project has realised that there were things that work well in layers to allow them to operate together. There were also 'anti-patterns' which stopped layers working well together.
+
+The intent of the compatibility programme is simple, if the layer passes the compatibility tests, it is considered “well behaved” and should operate and cooperate well with other compatible layers.
+
+The benefits of compatibility apply from multiple different user and member perspectives. From a hardware perspective (a BSP layer), compatibility means the hardware can be used in many different products and use cases without impacting the software stacks being run with it. For a company developing a product, compatibility gives you a specification/standard you can require in a contract and then know it will have certain desired characteristics for interoperability. It also puts constraints on how invasive these code bases are into the rest of the system, meaning that multiple different separate hardware support layers can coexist (e.g. for multiple product lines from different manufacturers). This can also influence how easily those system components might be upgraded or maintained for security purposes by one or more parties during the lifecycle of a product through development and widespread use.
+
+==================
+Validating a layer
+==================
+
+The badges are available to members of the project or open source projects run on a non-commercial basis and are tried to the project member level as a member benefit but anyone can answer the questions and run the script.
+
+The project encourages all layer maintainers to consider the questions and the output from the script against their layer as there are often unintentional consequences of the way some layers are constructed and the questions and script are designed to highlight commonly known issues which can often easily be solved leading to easier and wider layer use.
+
+It is intended that over time, the tests will evolve as best known practices are identified and as new interoperability issues are identified which unnecessarily restrict layer interoperability. If anyone becomes aware of either issue, please do mention it to the project (in our tech calls, on the mailing list or to the TSC). The TSC is holds overall responsibility for the technical criteria used by the programme.
+
+Layers are divided into three types:
+
+* "BSP" or "hardware support" layers contain support for particular pieces of hardware. This would include kernel and boot loader configuration, any recipes needed for firmware/modules needed for the hardware. They would usually correspond to a MACHINE setting.
+
+* "distro" layers defined as layers providing configuration options and settings such as a choice of init system, compiler/optimisation options or configuration and choices of software components. This would usually correspond to a DSITRO setting.
+
+* "software" layers are usually recipes. A layer might target a particular graphical UI or software stack component.
+
+Key best practises the programme tries to encourage:
+
+* A layer should clearly show who maintains it, where change submissions and bug reports should be sent
+* Where multiple types of functionality are present, the layer be internally subdivided into layers to separate these components as users would likely want to use only one of them and separability is a key best practise.
+* Adding a layer to a build should not change that build without the user taking some additional step of configuration to active the layer (such as selecting a MACHINE, a DISTRO or a DISTRO_FEATURE)
+
+The project does test the compatibility status of the core project layers on the project autobuilder.
+
+The official form to submit compatibility requests with is at https://www.yoctoproject.org/ecosystem/branding/compatible-registration/. Successful applicants can display the appropraiate badge which would be provided to them on success of the application.
+

meta/classes/sstate.bbclass

@@ -319,6 +319,8 @@ def sstate_install(ss, d):
     if os.path.exists(i):
         with open(i, "r") as f:
             manifests = f.readlines()
+    # We append new entries, we don't remove older entries which may have the same
+    # manifest name but different versions from stamp/workdir. See below.
     if filedata not in manifests:
         with open(i, "a+") as f:
             f.write(filedata)
@@ -1183,11 +1185,21 @@ python sstate_eventhandler_reachablestamps() {
         i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
         if not os.path.exists(i):
             continue
+        manseen = set()
+        ignore = []
         with open(i, "r") as f:
             lines = f.readlines()
-            for l in lines:
+            for l in reversed(lines):
                 try:
                     (stamp, manifest, workdir) = l.split()
+                    # The index may have multiple entries for the same manifest as the code above only appends
+                    # new entries and there may be an entry with matching manifest but differing version in stamp/workdir.
+                    # The last entry in the list is the valid one, any earlier entries with matching manifests
+                    # should be ignored.
+                    if manifest in manseen:
+                        ignore.append(l)
+                        continue
+                    manseen.add(manifest)
                     if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
                         toremove.append(l)
                         if stamp not in seen:
@@ -1218,6 +1230,8 @@ python sstate_eventhandler_reachablestamps() {
 
         with open(i, "w") as f:
             for l in lines:
+                if l in ignore:
+                    continue
                 f.write(l)
     machineindex |= set(stamps)
     with open(mi, "w") as f:

meta/conf/distro/include/cve-extra-exclusions.inc

@@ -0,0 +1,103 @@
+# This file contains a list of CVE's where resolution has proven to be impractical.
+# It contains all the information we are aware of about an issue and analysis about
+# why we believe it can't be fixed/handled. Additional information is welcome through
+# patches to the file.
+#
+# Include this file in your local.conf or distro.conf to exclude these CVE's
+# from the cve-check results
+
+# strace https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2000-0006
+# CVE is more than 20 years old with no resolution evident
+# broken links in CVE database references make resolution impractical
+CVE_CHECK_WHITELIST += "CVE-2000-0006"
+
+# epiphany https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2005-0238
+# The issue here is spoofing of domain names using characters from other character sets.
+# There has been much discussion amongst the epiphany and webkit developers and
+# whilst there are improvements about how domains are handled and displayed to the user
+# there is unlikely ever to be a single fix to webkit or epiphany which addresses this
+# problem. Whitelisted as there isn't any mitigation or fix or way to progress this further
+# we can seem to take.
+CVE_CHECK_WHITELIST += "CVE-2005-0238"
+
+# glibc https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2010-4756
+# Issue is memory exhaustion via glob() calls, e.g. from within an ftp server
+# Best discussion in https://bugzilla.redhat.com/show_bug.cgi?id=681681
+# Upstream don't see it as a security issue, ftp servers shouldn't be passing
+# this to libc glob. Exclude as upstream have no plans to add BSD's GLOB_LIMIT or similar
+CVE_CHECK_WHITELIST += "CVE-2010-4756"
+
+# go https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2020-29509
+# go https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2020-29511
+# The encoding/xml package in go can potentially be used for security exploits if not used correctly
+# CVE applies to a netapp product as well as flagging a general issue. We don't ship anything
+# exposing this interface in an exploitable way
+CVE_CHECK_WHITELIST += "CVE-2020-29509 CVE-2020-29511"
+
+
+
+#### Upstream still working on ####
+
+# qemu:qemu-native:qemu-system-native https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-20255
+# There was a proposed patch https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg06098.html
+# however qemu maintainers are sure the patch is incorrect and should not be applied.
+
+# flex:flex-native https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2019-6293
+# Upstream bug, still open: https://github.com/westes/flex/issues/414
+# Causes memory exhaustion so potential DoS but no buffer overflow, low priority
+
+# wget https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-31879
+# https://mail.gnu.org/archive/html/bug-wget/2021-02/msg00002.html
+# No response upstream as of 2021/5/12
+
+# bind https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-25215
+# Fix not in upstream release yet as of 2021/5/12
+
+
+
+
+#### CPE update pending ####
+
+# groff:groff-native https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2000-0803
+# Appears it was fixed in https://git.savannah.gnu.org/cgit/groff.git/commit/?id=07f95f1674217275ed4612f1dcaa95a88435c6a7
+# so from 1.17 onwards. Reported to the database for update by RP 2021/5/9. Update accepted 2021/5/10.
+#CVE_CHECK_WHITELIST += "CVE-2000-0803"
+
+# grub:grub-efi:grub-native https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2019-14865
+# Looks like grub-set-bootflag is patched in by Fedora/RHEL:
+# https://src.fedoraproject.org/rpms/grub2/blob/498ea7003b4dd8079fc075fad7e19e0b190d0f97/f/0133-Add-grub-set-bootflag-utility.patch
+# Does not exist in upstream grub2:
+# https://git.savannah.gnu.org/cgit/grub.git/tree/util
+# Reported to the database for update by RP 2021/5/9 Update accepted 2021/5/12
+#CVE_CHECK_WHITELIST += "CVE-2019-14865"
+
+#qemu:qemu-native:qemu-system-native https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2008-4539
+# Fixed in https://git.qemu.org/?p=qemu.git;a=commit;h=65d35a09979e63541afc5bfc595b9f1b1b4ae069
+# which was in 0.10.0. Reported to the database for update by RP 2021/5/10
+#CVE_CHECK_WHITELIST += "CVE-2008-4539" Update accepted 2021/5/11
+
+# coreutils:coreutils-native https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2013-0221
+# coreutils:coreutils-native https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2013-0222
+# coreutils:coreutils-native https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2013-0223
+# Patch with issues is SUSE/RHEL specific. RP has suggested a CPE update to the database 2021/5/10 Update accepted 2021/5/11
+#CVE_CHECK_WHITELIST += "CVE-2013-0221 CVE-2013-0222 CVE-2013-0223"
+
+# libgit2 https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2014-9390
+# Only applies to libgit2 versions up to 0.12.2. RP requested CPE entry update. Update accepted 2021/5/12
+#CVE_CHECK_WHITELIST += "CVE-2014-9390"
+
+# bind https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2017-3139
+# https://bugzilla.redhat.com/show_bug.cgi?id=1447743
+# "This issue did not affect any upstream versions of BIND."
+# RP sent CPE entry change request as RHEL only Update accepted 2021/5/12
+#CVE_CHECK_WHITELIST += "CVE-2017-3139"
+
+# libexif https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-6328
+# Ross sent CPE change upstream to database
+#CVE_CHECK_WHITELIST += "CVE-2016-6328"
+
+
+
+
+
+

meta/conf/distro/include/tcmode-default.inc

@@ -22,7 +22,7 @@ BINUVERSION ?= "2.36%"
 GDBVERSION ?= "10.%"
 GLIBCVERSION ?= "2.33"
 LINUXLIBCVERSION ?= "5.10%"
-QEMUVERSION ?= "5.2%"
+QEMUVERSION ?= "6.0%"
 GOVERSION ?= "1.16%"
 # This can not use wildcards like 8.0.% since it is also used in mesa to denote
 # llvm version being used, so always bump it with llvm recipe version bump