[Buildstream SDK] Update to FDO 23.08 release

https://bugs.webkit.org/show_bug.cgi?id=262879

Reviewed by Carlos Alberto Lopez Perez and Carlos Garcia Campos.

This new SDK version notably ships:

- ICU 73.2
- GCC 13.2.0
- clang 16.0.6
- sccache 0.7.1
- Mesa 23.1.8
- Cairo 1.18.0
- Ruby 3.2.2
- Python 3.11.5

The SDK build infrastructure was migrated from Pipenv to a Podman container shipping BuildStream2.
This new approach should make the build easier to setup.

The ccls recipe was removed, clangd is now preferred. The GStreamer kate plugin was removed,
upstream has also removed it and it won't ship in GStreamer 1.24.

This patch updates only the build recipes. The updated SDK will be deployed on the bots later on.

* .gitignore:
* Tools/Scripts/bst-wrapper:
(run):
(ensure_environment):
(main):
(run_pipenv): Deleted.
(run_bst): Deleted.
* Tools/Scripts/webkit-flatpak-sdk:
(run_wrapper):
* Tools/buildstream/Makefile:
* Tools/buildstream/Pipfile: Removed.
* Tools/buildstream/Pipfile.lock: Removed.
* Tools/buildstream/README.md:
* Tools/buildstream/elements/flatpak-platform-extensions-extra.bst:
* Tools/buildstream/elements/flatpak-platform-extensions.bst:
* Tools/buildstream/elements/flatpak-runtimes.bst:
* Tools/buildstream/elements/freedesktop-sdk.bst:
* Tools/buildstream/elements/plugins/bst-plugins-experimental.bst: Added.
* Tools/buildstream/elements/plugins/buildstream-plugins.bst: Added.
* Tools/buildstream/elements/qt5/qtbase.bst:
* Tools/buildstream/elements/qt5/qtdeclarative.bst:
* Tools/buildstream/elements/qt5/qtquickcontrols.bst:
* Tools/buildstream/elements/qt5/qtquickcontrols2.bst:
* Tools/buildstream/elements/qt5/qtwayland.bst:
* Tools/buildstream/elements/qt5/qtx11extras.bst:
* Tools/buildstream/elements/sdk-build-depends/rapidjson.bst: Removed.
* Tools/buildstream/elements/sdk-build-depends/scons.bst:
* Tools/buildstream/elements/sdk-platform.bst:
* Tools/buildstream/elements/sdk.bst:
* Tools/buildstream/elements/sdk/apitrace.bst:
* Tools/buildstream/elements/sdk/breakpad.bst:
* Tools/buildstream/elements/sdk/capnp.bst:
* Tools/buildstream/elements/sdk/cargo-c.bst:
* Tools/buildstream/elements/sdk/ccls.bst: Removed.
* Tools/buildstream/elements/sdk/clangd.bst:
* Tools/buildstream/elements/sdk/eigen.bst:
* Tools/buildstream/elements/sdk/flite.bst:
* Tools/buildstream/elements/sdk/gi-docgen.bst:
* Tools/buildstream/elements/sdk/gst-plugin-closedcaption.bst:
* Tools/buildstream/elements/sdk/gst-plugin-dav1d.bst:
* Tools/buildstream/elements/sdk/gst-plugin-rtp.bst:
* Tools/buildstream/elements/sdk/gtk+-3.bst:
* Tools/buildstream/elements/sdk/gtk.bst:
* Tools/buildstream/elements/sdk/icecc.bst:
* Tools/buildstream/elements/sdk/libavif.bst:
* Tools/buildstream/elements/sdk/libbacktrace.bst:
* Tools/buildstream/elements/sdk/libevent.bst:
* Tools/buildstream/elements/sdk/libjxl.bst:
* Tools/buildstream/elements/sdk/libmanette.bst:
* Tools/buildstream/elements/sdk/libusrsctp.bst:
* Tools/buildstream/elements/sdk/libwpe.bst:
* Tools/buildstream/elements/sdk/mold.bst:
* Tools/buildstream/elements/sdk/monado.bst:
* Tools/buildstream/elements/sdk/openxr.bst:
* Tools/buildstream/elements/sdk/patchelf.bst:
* Tools/buildstream/elements/sdk/psmisc.bst:
* Tools/buildstream/elements/sdk/python3-smartypants.bst:
* Tools/buildstream/elements/sdk/rr.bst:
* Tools/buildstream/elements/sdk/sccache.bst:
* Tools/buildstream/elements/sdk/sparkle-cdm.bst:
* Tools/buildstream/elements/sdk/svt-av1.bst:
* Tools/buildstream/elements/sdk/tbb.bst:
* Tools/buildstream/elements/sdk/wpebackend-fdo.bst:
* Tools/buildstream/elements/sdk/xdg-dbus-proxy.bst:
* Tools/buildstream/elements/test-infra.bst:
* Tools/buildstream/elements/test-infra/apr.bst:
* Tools/buildstream/elements/test-infra/httpd.bst:
* Tools/buildstream/elements/test-infra/perl-digest-crc-pm.bst: Added.
* Tools/buildstream/elements/test-infra/webkitgtk-test-dicts.bst:
* Tools/buildstream/patches/fdo-0001-gst-plugins-bad-Add-support-Kate.patch: Removed.
* Tools/buildstream/patches/fdo-0001-mesa-Bump-to-23.1.5.patch: Removed.
* Tools/buildstream/patches/fdo-0001-pipewire-base-Disable-AEC-module.patch: Removed.
* Tools/buildstream/patches/fdo-0002-ffmpeg-Support-more-codecs.patch:
* Tools/buildstream/patches/fdo-0003-gst-plugins-bad-Support-for-updated-webrtc-audio-pro.patch: Removed.
* Tools/buildstream/patches/fdo-0004-gst-plugins-ugly-Enable-x264-encoder.patch:
* Tools/buildstream/patches/fdo-0005-GStreamer-Bump-to-1.22.6.patch:
* Tools/buildstream/patches/fdo-0006-gst-plugins-bad-Enable-soundtouch.patch:
* Tools/buildstream/patches/fdo-0007-components-Bump-GLib-to-version-2.76.patch: Removed.
* Tools/buildstream/patches/fdo-0008-components-Bump-g-i-to-version-1.76.patch: Removed.
* Tools/buildstream/patches/fdo-0009-gst-plugins-bad-Enable-x265-encoder.patch:
* Tools/buildstream/patches/flite-fix-voice-list-build.patch: Added.
* Tools/buildstream/patches/rapidjson-0001-Fix-build-warnings-emitted-by-GCC-10-on-Aarch64.patch: Removed.
* Tools/buildstream/patches/tbb-Retry-if-pthread_create-fails-with-EAGAIN.patch: Removed.
* Tools/buildstream/project.conf:

Canonical link: https://commits.webkit.org/270440@main

.gitignore

@@ -14,8 +14,7 @@ DerivedData
 results
 
 # Ignore Buildstream local files
-/Tools/buildstream/.bst
-/Tools/buildstream/.venv
+/Tools/buildstream/.bst2
 /Tools/buildstream/cache
 /Tools/buildstream/flatpak-version.yml
 /Tools/buildstream/repo

Tools/Scripts/bst-wrapper

@@ -23,31 +23,37 @@ import sys
 
 scriptdir = os.path.abspath(os.path.dirname(__file__))
 SDK_PROJECT_PATH = os.path.join(scriptdir, "..", "buildstream")
-PIPENV_PIPFILE = os.path.realpath(os.path.join(SDK_PROJECT_PATH, "Pipfile"))
 
-def run_pipenv(args: list, **kw) -> subprocess.CompletedProcess:
-    local_env = {
-        "PIPENV_VENV_IN_PROJECT": "1",
-        "PIPENV_PIPFILE": PIPENV_PIPFILE
-    }
-    kw["env"] = os.environ
-    kw["env"].update(local_env)
-    return subprocess.run(["pipenv", ] + args, **kw)
+# DOCKER_IMAGE_ID from https://gitlab.com/freedesktop-sdk/freedesktop-sdk/-/blob/master/.gitlab-ci.yml
+IMAGE_ID = 'f2004eceea4b91b56e761826248274b2f66d28b3'
 
-def ensure_environment():
-    result = run_pipenv(["--venv",], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    if result.returncode != 0:
-        run_pipenv(["install"])
+TOOLBOX_NAME = f'bst2-{IMAGE_ID}'
+IMAGE = f'registry.gitlab.com/freedesktop-sdk/infrastructure/freedesktop-sdk-docker-images/bst2:{IMAGE_ID}'
+
+def run(*args, in_container=False, capture_output=True):
+    if in_container:
+        cmdline = f'toolbox run -c {TOOLBOX_NAME}'.split() + list(args)
+    else:
+        cmdline = args
+    print("Running %s" % " ".join(cmdline))
+    proc = subprocess.run(cmdline, capture_output=capture_output, text=True)
+    if proc.returncode != 0:
+        raise Exception(proc.returncode)
+    if capture_output:
+        return proc.stdout.strip()
 
-def run_bst(args: list) -> subprocess.CompletedProcess:
-    return run_pipenv(["run", "bst", "-C", SDK_PROJECT_PATH] + args)
+def ensure_environment():
+    all_containers = run('podman', 'container', 'list', '-a', '--format', '{{.Names}}').splitlines()
+    if TOOLBOX_NAME not in all_containers:
+        run('toolbox', 'create', '-c', TOOLBOX_NAME, '--image', IMAGE, '-y', capture_output=False)
 
 def main(args: list) -> int:
     ensure_environment()
-    if args[0] == "pipenv":
-        run_pipenv(args[1:])
+    if args[0] == "toolbox":
+        filtered_args = args[1:]
     else:
-        run_bst(args)
+        filtered_args = ["bst", "-C", SDK_PROJECT_PATH] + args
+    run(*filtered_args, in_container=True, capture_output=False)
     return 0
 
 if __name__ == "__main__":

Tools/Scripts/webkit-flatpak-sdk

@@ -26,9 +26,8 @@ scriptdir = os.path.abspath(os.path.dirname(__file__))
 SDK_PROJECT_PATH = os.path.join(scriptdir, "..", "buildstream")
 wrapper = os.path.join(scriptdir, "bst-wrapper")
 
-def run_wrapper(args: list, **kw) -> subprocess.CompletedProcess:
-    kw["cwd"] = SDK_PROJECT_PATH
-    return subprocess.run(["make", f"BST={wrapper}",] + args, **kw)
+def run_wrapper(args: list) -> subprocess.CompletedProcess:
+    return subprocess.run(["make", "-C", SDK_PROJECT_PATH, f"BST={wrapper}",] + args)
 
 def main(args: list) -> int:
     parser = argparse.ArgumentParser()

Tools/buildstream/Makefile

@@ -1,145 +1,37 @@
 ARCH ?= $(shell flatpak --default-arch)
 BOOTSTRAP_ARCH ?= $(shell flatpak --default-arch)
 
-GIT ?= git
-
 BST ?= bst
 BST_ARGS ?=
 _BST_ARGS ?= --no-interactive -o arch $(ARCH) -o bootstrap_build_arch $(BOOTSTRAP_ARCH)
 
-OSTREE ?= ostree
-
-CACHEDIR ?= cache
-
 REPO ?= repo
-GPG_KEY = 9A0495AF96828F9D5E032C46A9A60744BCE3F878
-
-# EXPORT_ARGS ?= --gpg-sign=${GPG_KEY} --gpg-homedir=gpg
 EXPORT_ARGS ?=
 
 RSYNC_HOST ?= software.igalia.com
 RSYNC_REMOTE_DIR ?= /var/www/software/webkit-sdk-repo
 RSYNC_ARGS ?= -vz
 
-FLATPAK_RUNTIMES_REPO = $(CACHEDIR)/flatpak-runtimes-repo
-FLATPAK_PLATFORM_EXTENSIONS_REPO = $(CACHEDIR)/flatpak-platform-extensions-repo
-FLATPAK_PLATFORM_EXTENSIONS_EXTRA_REPO = $(CACHEDIR)/flatpak-platform-extensions-extra-repo
-EXPORT_REPO = $(REPO)
-
-EXPORT_REFS = $(shell [ -d "$(EXPORT_REPO)" ] && $(OSTREE) refs --repo $(EXPORT_REPO))
-
-all: export
-
-clean:
-	if [ -d "$(CACHEDIR)" ]; then rmdir $(CACHEDIR); fi
-	if [ -d "$(EXPORT_REPO)" ]; then rm -r $(EXPORT_REPO); fi
-.PHONY: clean
-
-export: | $(EXPORT_REPO)
-ifneq ($(EXPORT_ARGS),)
-	flatpak build-sign $(EXPORT_ARGS) $|
-endif
-	flatpak build-update-repo --generate-static-deltas $(EXPORT_ARGS) $|
-.PHONY: export
-
-
-fetch-junctions:
-	$(BST) $(BST_ARGS) $(_BST_ARGS) fetch freedesktop-sdk.bst
-.PHONY: fetch-junctions
-
-track:
-	$(BST) $(BST_ARGS) $(_BST_ARGS) track freedesktop-sdk.bst
-	$(BST) $(BST_ARGS) $(_BST_ARGS) track flatpak-runtimes.bst flatpak-platform-extensions.bst flatpak-platform-extensions-extra.bst --deps=all
-.PHONY: track
-
-
-$(CACHEDIR):
-	mkdir -p $@
-
-$(EXPORT_REPO):
-	$(OSTREE) init --repo=$@ --mode=archive
-
-
-.git/hooks/pre-commit: utils/git-pre-commit
-	ln -frs $< .git/hooks/pre-commit
-
-
-flatpak-version.yml: $(GIT_HOOKS) | CLEAN-flatpak-version.yml
-	./utils/generate-version $@
-.PHONY: flatpak-version.yml
-
-CLEAN-flatpak-version.yml:
-	rm -f flatpak-version.yml
-.PHONY: CLEAN-flatpak-version.yml
-clean: CLEAN-flatpak-version.yml
-
-
-BUILD-flatpak-runtimes: flatpak-version.yml elements/**/*.bst
+flatpak-repo:
+	test -e $(REPO) || ostree init --repo=$(REPO) --mode=archive
+	./utils/generate-version flatpak-version.yml
 	$(BST) $(BST_ARGS) $(_BST_ARGS) build flatpak-runtimes.bst
-.PHONY: BUILD-flatpak-runtimes
-
-CHECK-flatpak-runtimes: flatpak-version.yml | fetch-junctions
-	$(BST) $(BST_ARGS) $(_BST_ARGS) show flatpak-runtimes.bst
-.PHONY: CHECK-flatpak-runtimes
-check: CHECK-flatpak-runtimes
-
-$(FLATPAK_RUNTIMES_REPO): BUILD-flatpak-runtimes | $(CACHEDIR)
-	$(BST) $(BST_ARGS) $(_BST_ARGS) checkout --hardlinks --force flatpak-runtimes.bst $@
-
-EXPORT-$(FLATPAK_RUNTIMES_REPO): $(FLATPAK_RUNTIMES_REPO) | $(EXPORT_REPO)
-	$(OSTREE) pull-local --repo=$| $<
-.PHONY: EXPORT-$(FLATPAK_RUNTIMES_REPO)
-export: EXPORT-$(FLATPAK_RUNTIMES_REPO)
-
-CLEAN-$(FLATPAK_RUNTIMES_REPO):
-	rm -rf $(FLATPAK_RUNTIMES_REPO)
-.PHONY: CLEAN-$(FLATPAK_RUNTIMES_REPO)
-clean: CLEAN-$(FLATPAK_RUNTIMES_REPO)
-
-
-BUILD-flatpak-platform-extensions: elements/**/*.bst
 	$(BST) $(BST_ARGS) $(_BST_ARGS) build flatpak-platform-extensions.bst
-.PHONY: BUILD-flatpak-platform-extensions
-
-CHECK-flatpak-platform-extensions: | fetch-junctions
-	$(BST) $(BST_ARGS) $(_BST_ARGS) show flatpak-platform-extensions.bst
-.PHONY: CHECK-flatpak-platform-extensions
-check: CHECK-flatpak-platform-extensions
-
-$(FLATPAK_PLATFORM_EXTENSIONS_REPO): BUILD-flatpak-platform-extensions | $(CACHEDIR)
-	$(BST) $(BST_ARGS) $(_BST_ARGS) checkout --hardlinks --force flatpak-platform-extensions.bst $@
-
-EXPORT-$(FLATPAK_PLATFORM_EXTENSIONS_REPO): $(FLATPAK_PLATFORM_EXTENSIONS_REPO) | $(EXPORT_REPO)
-	$(OSTREE) pull-local --repo=$| $<
-.PHONY: EXPORT-$(FLATPAK_PLATFORM_EXTENSIONS_REPO)
-export: EXPORT-$(FLATPAK_PLATFORM_EXTENSIONS_REPO)
-
-CLEAN-$(FLATPAK_PLATFORM_EXTENSIONS_REPO):
-	rm -rf $(FLATPAK_PLATFORM_EXTENSIONS_REPO)
-.PHONY: CLEAN-$(FLATPAK_PLATFORM_EXTENSIONS_REPO)
-clean: CLEAN-$(FLATPAK_PLATFORM_EXTENSIONS_REPO)
-
-BUILD-flatpak-platform-extensions-extra: elements/**/*.bst
 	$(BST) $(BST_ARGS) $(_BST_ARGS) build flatpak-platform-extensions-extra.bst
-.PHONY: BUILD-flatpak-platform-extensions-extra
-
-CHECK-flatpak-platform-extensions-extra: | fetch-junctions
-	$(BST) $(BST_ARGS) $(_BST_ARGS) show flatpak-platform-extensions-extra.bst
-.PHONY: CHECK-flatpak-platform-extensions-extra
-check: CHECK-flatpak-platform-extensions-extra
-
-$(FLATPAK_PLATFORM_EXTENSIONS_EXTRA_REPO): BUILD-flatpak-platform-extensions-extra | $(CACHEDIR)
-	$(BST) $(BST_ARGS) $(_BST_ARGS) checkout --hardlinks --force flatpak-platform-extensions-extra.bst $@
-
-EXPORT-$(FLATPAK_PLATFORM_EXTENSIONS_EXTRA_REPO): $(FLATPAK_PLATFORM_EXTENSIONS_EXTRA_REPO) | $(EXPORT_REPO)
-	$(OSTREE) pull-local --repo=$| $<
-.PHONY: EXPORT-$(FLATPAK_PLATFORM_EXTENSIONS_EXTRA_REPO)
-export: EXPORT-$(FLATPAK_PLATFORM_EXTENSIONS_EXTRA_REPO)
-
-CLEAN-$(FLATPAK_PLATFORM_EXTENSIONS_EXTRA_REPO):
-	rm -rf $(FLATPAK_PLATFORM_EXTENSIONS_EXTRA_REPO)
-.PHONY: CLEAN-$(FLATPAK_PLATFORM_EXTENSIONS_EXTRA_REPO)
-clean: CLEAN-$(FLATPAK_PLATFORM_EXTENSIONS_EXTRA_REPO)
+	rm -rf checkout-repo/
+	$(BST) $(BST_ARGS) $(_BST_ARGS) artifact checkout flatpak-runtimes.bst --directory checkout-repo/
+	ostree pull-local --repo $(REPO) checkout-repo/
+	rm -rf checkout-repo/
+	$(BST) $(BST_ARGS) $(_BST_ARGS) artifact checkout flatpak-platform-extensions.bst --directory checkout-repo/
+	ostree pull-local --repo $(REPO) checkout-repo/
+	rm -rf checkout-repo/
+	$(BST) $(BST_ARGS) $(_BST_ARGS) artifact checkout flatpak-platform-extensions-extra.bst --directory checkout-repo/
+	ostree pull-local --repo $(REPO) checkout-repo/
+	rm -rf checkout-repo/
+ifneq ($(EXPORT_ARGS),)
+	flatpak build-sign $(EXPORT_ARGS) $(REPO)
+	flatpak build-update-repo --generate-static-deltas $(EXPORT_ARGS) $(REPO)
+endif
 
 pull-repo:
 	./ostree-releng-scripts/rsync-repos --rsync-opts="${RSYNC_ARGS}" --src ${RSYNC_HOST}:${RSYNC_REMOTE_DIR}/ --dest ${REPO}