builder-manifest: Run appstream-compose with no-partial-urls argument

.github/workflows/check.yml

@@ -74,14 +74,14 @@ jobs:
       run: make -j $(getconf _NPROCESSORS_ONLN)
 
   check-meson:
-    name: Ubuntu 22.04 meson build
-    runs-on: ubuntu-22.04
+    name: Ubuntu meson build
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: ['ubuntu-22.04', 'ubuntu-24.04']
         compiler: ['gcc', 'clang']
 
     env:
-      UBUNTU_VERSION: '22.04'
       CC: ${{ matrix.compiler }}
       BASE_CFLAGS: -Wp,-D_FORTIFY_SOURCE=2
       BUILDDIR: builddir

doc/flatpak-builder.xml

@@ -589,6 +589,21 @@
                 </para></listitem>
             </varlistentry>
 
+            <varlistentry>
+                <term><option>--compose-url-policy=POLICY</option></term>
+
+                <listitem><para>
+                  Set the AppStream compose URL policy. Accepted values
+                  are <literal>partial</literal> and <literal>full</literal>.
+                  <literal>full</literal> requires AppStream version >= 0.16.3.
+                  Defaults to <literal>partial</literal> if unspecified.
+                  This policy only takes effect when used in conjunction
+                  with <option>--mirror-screenshots-url=URL</option>;
+                  otherwise the Appstream catalogue will preserve
+                  the source media URLs.
+                </para></listitem>
+            </varlistentry>
+
             <varlistentry>
                 <term><option>--add-tag=TAG</option></term>
 

src/builder-context.c

@@ -88,6 +88,8 @@ struct BuilderContext
   char           *opt_mirror_screenshots_url;
 
   BuilderSdkConfig *sdk_config;
+
+  BuilderAsUrlPolicy as_url_policy;
 };
 
 typedef struct
@@ -1241,6 +1243,19 @@ builder_context_create_state_dir (BuilderContext *self,
   return TRUE;
 }
 
+void
+builder_context_set_as_url_policy (BuilderContext     *self,
+                                   BuilderAsUrlPolicy  policy)
+{
+  self->as_url_policy = policy;
+}
+
+BuilderAsUrlPolicy
+builder_context_get_as_url_policy (BuilderContext *self)
+{
+  return self->as_url_policy;
+}
+
 BuilderContext *
 builder_context_new (GFile *run_dir,
                      GFile *app_dir,

src/builder-context.h

@@ -29,6 +29,11 @@
 
 G_BEGIN_DECLS
 
+typedef enum {
+  BUILDER_AS_URL_POLICY_PARTIAL = 0,
+  BUILDER_AS_URL_POLICY_FULL,
+} BuilderAsUrlPolicy;
+
 /* Same as SOUP_HTTP_URI_FLAGS, means all possible flags for http uris */
 
 #if GLIB_CHECK_VERSION (2, 68, 0)
@@ -185,6 +190,10 @@ BuilderSdkConfig * builder_context_get_sdk_config (BuilderContext *self);
 gboolean        builder_context_create_state_dir (BuilderContext *self,
                                                   GError        **error);
 
+void             builder_context_set_as_url_policy (BuilderContext      *self,
+                                                    BuilderAsUrlPolicy  policy);
+BuilderAsUrlPolicy builder_context_get_as_url_policy (BuilderContext   *self);
+
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (BuilderContext, g_object_unref)
 
 G_END_DECLS

src/builder-main.c

@@ -89,6 +89,7 @@ static gboolean opt_log_session_bus;
 static gboolean opt_log_system_bus;
 static gboolean opt_yes;
 static gint64 opt_source_date_epoch = -1;
+static gchar *opt_as_url_policy = NULL;
 
 static GOptionEntry entries[] = {
   { "verbose", 'v', 0, G_OPTION_ARG_NONE, &opt_verbose, "Print debug information during command processing", NULL },
@@ -144,6 +145,7 @@ static GOptionEntry entries[] = {
   { "assumeyes", 'y', 0, G_OPTION_ARG_NONE, &opt_yes, N_("Automatically answer yes for all questions"), NULL },
   { "no-shallow-clone", 0, 0, G_OPTION_ARG_NONE, &opt_no_shallow_clone, "Don't use shallow clones when mirroring git repos", NULL },
   { "override-source-date-epoch", 0, 0, G_OPTION_ARG_INT64, &opt_source_date_epoch, "Use this timestamp to perform the build, instead of the last modification time of the manifest.", NULL },
+  { "compose-url-policy", 0, 0, G_OPTION_ARG_STRING, &opt_as_url_policy, "Set the AppStream compose URL policy to either 'partial' (default) or 'full'", "POLICY" },
   { NULL }
 };
 
@@ -608,6 +610,29 @@ main (int    argc,
   builder_context_set_opt_export_only (build_context, opt_export_only);
   builder_context_set_opt_mirror_screenshots_url (build_context, opt_mirror_screenshots_url);
 
+  if (opt_mirror_screenshots_url)
+    {
+      BuilderAsUrlPolicy policy = BUILDER_AS_URL_POLICY_PARTIAL;
+
+      if (g_strcmp0 (opt_as_url_policy, "full") == 0)
+        policy = BUILDER_AS_URL_POLICY_FULL;
+      else if (g_strcmp0 (opt_as_url_policy, "partial") == 0)
+        policy = BUILDER_AS_URL_POLICY_PARTIAL;
+      else if (opt_as_url_policy != NULL)
+        {
+          g_printerr ("Invalid value for --compose-url-policy: %s\n", opt_as_url_policy);
+          return 1;
+        }
+
+      if (policy == BUILDER_AS_URL_POLICY_FULL && !appstream_has_version (0, 16, 3))
+        {
+          g_printerr ("AppStream version >= 0.16.3 required for 'full' compose URL policy\n");
+          return 1;
+        }
+
+      builder_context_set_as_url_policy (build_context, policy);
+    }
+
   git_init_email ();
 
   if (!builder_context_create_state_dir (build_context, &error))

src/builder-manifest.c

@@ -2422,7 +2422,8 @@ cmpstringp (const void *p1, const void *p2)
 }
 
 static gboolean
-appstreamcli_compose (GError **error,
+appstreamcli_compose (GError             **error,
+                      BuilderAsUrlPolicy   as_url_policy,
                       ...)
 {
   g_autoptr(GPtrArray) args = NULL;
@@ -2433,7 +2434,10 @@ appstreamcli_compose (GError **error,
   g_ptr_array_add (args, g_strdup ("appstreamcli"));
   g_ptr_array_add (args, g_strdup ("compose"));
 
-  va_start (ap, error);
+  if (as_url_policy == BUILDER_AS_URL_POLICY_FULL)
+    g_ptr_array_add (args, g_strdup ("--no-partial-urls"));
+
+  va_start (ap, as_url_policy);
   while ((arg = va_arg (ap, const gchar *)))
     g_ptr_array_add (args, g_strdup (arg));
   g_ptr_array_add (args, NULL);
@@ -3066,6 +3070,7 @@ builder_manifest_cleanup (BuilderManifest *self,
                                                        flatpak_file_get_path_cached (icon_out));
           const char *opt_mirror_screenshots_url = builder_context_get_opt_mirror_screenshots_url (context);
           gboolean opt_export_only = builder_context_get_opt_export_only (context);
+          BuilderAsUrlPolicy as_url_policy = builder_context_get_as_url_policy (context);
 
           if (opt_mirror_screenshots_url && !opt_export_only)
             {
@@ -3077,6 +3082,7 @@ builder_manifest_cleanup (BuilderManifest *self,
               g_print ("Running appstreamcli compose\n");
               g_print ("Saving screenshots in %s\n", flatpak_file_get_path_cached (media_dir));
               if (!appstreamcli_compose (error,
+                                         as_url_policy,
                                          "--prefix=/",
                                          origin,
                                          arg_base_url,
@@ -3093,6 +3099,7 @@ builder_manifest_cleanup (BuilderManifest *self,
             {
               g_print ("Running appstreamcli compose\n");
               if (!appstreamcli_compose (error,
+                                         as_url_policy,
                                          "--prefix=/",
                                          origin,
                                          result_root_arg,

src/builder-utils.c

@@ -1837,3 +1837,52 @@ flatpak_version_check (int major,
 
   return FALSE;
 }
+
+gboolean
+appstream_has_version (int major,
+                       int minor,
+                       int micro)
+{
+  static int as_major = 0;
+  static int as_minor = 0;
+  static int as_micro = 0;
+
+  if (as_major == 0 &&
+      as_minor == 0 &&
+      as_micro == 0)
+    {
+      const char * argv[] = { "appstreamcli", "--version", NULL };
+      g_autoptr(GSubprocess) subp = NULL;
+      g_autofree char *out = NULL;
+      g_auto(GStrv) lines = NULL;
+
+      subp = g_subprocess_newv (argv, G_SUBPROCESS_FLAGS_STDOUT_PIPE, NULL);
+      g_subprocess_communicate_utf8 (subp, NULL, NULL, &out, NULL, NULL);
+
+      lines = g_strsplit (out, "\n", -1);
+
+      for (size_t i = 0; lines[i] != NULL; i++)
+        {
+          /* Only prefer library version over cli version in case of mismatch */
+          if (g_str_has_prefix (lines[i], "AppStream library version:"))
+            {
+              if (sscanf (lines[i], "AppStream library version: %d.%d.%d", &as_major, &as_minor, &as_micro) == 3)
+                break;
+            }
+          else if (g_str_has_prefix (lines[i], "AppStream version:"))
+            {
+              if (sscanf (lines[i], "AppStream version: %d.%d.%d", &as_major, &as_minor, &as_micro) == 3)
+                break;
+            }
+        }
+
+      if (as_major == 0 && as_minor == 0 && as_micro == 0)
+        g_warning ("Failed to find appstream version");
+      else
+        g_debug ("Found AppStream version %d.%d.%d", as_major, as_minor, as_micro);
+    }
+
+  return (as_major > major) ||
+         (as_major == major && as_minor > minor) ||
+         (as_major == major && as_minor == minor && as_micro >= micro);
+}

src/builder-utils.h

@@ -216,6 +216,10 @@ gboolean flatpak_version_check (int major,
                                 int minor,
                                 int micro);
 
+gboolean appstream_has_version (int major,
+                                int minor,
+                                int micro);
+
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakXml, flatpak_xml_free);
 
 G_END_DECLS

tests/Makefile.am.inc

@@ -61,6 +61,8 @@ dist_installed_test_data = \
 	tests/org.flatpak_builder.gui.desktop \
 	tests/org.flatpak_builder.gui.json \
 	tests/org.flatpak_builder.gui.metainfo.xml \
+	tests/org.flatpak.appstream_media.json \
+	tests/org.test.Hello-256.png \
 	$(NULL)
 
 installed_test_keyringdir = $(installed_testdir)/test-keyring

tests/libtest.sh

@@ -309,6 +309,47 @@ skip_without_python2 () {
     fi
 }
 
+appstream_has_version () {
+    req_major=$1
+    req_minor=$2
+    req_micro=$3
+
+    maj=0; min=0; mic=0
+
+    out=$(appstreamcli --version 2>/dev/null) || return 1
+
+    while IFS= read -r line; do
+        case "$line" in
+            "AppStream library version:"* )
+                ver=$(echo "$line" | awk '{print $4}')
+                ;;
+            "AppStream version:"* )
+                ver=$(echo "$line" | awk '{print $3}')
+                ;;
+            * ) continue ;;
+        esac
+
+        maj=$(echo "$ver" | cut -d. -f1)
+        min=$(echo "$ver" | cut -d. -f2)
+        mic=$(echo "$ver" | cut -d. -f3)
+        break
+    done <<EOF
+$out
+EOF
+
+    if [ "$maj" -gt "$req_major" ]; then
+        return 0
+    elif [ "$maj" -eq "$req_major" ]; then
+        if [ "$min" -gt "$req_minor" ]; then
+            return 0
+        elif [ "$min" -eq "$req_minor" ]; then
+            [ "$mic" -ge "$req_micro" ] && return 0
+        fi
+    fi
+
+    return 1
+}
+
 cleanup () {
     gpg-connect-agent --homedir "${FL_GPG_HOMEDIR}" killagent /bye >&2 || true
     if test -n "${TEST_SKIP_CLEANUP:-}"; then

tests/meson.build

@@ -108,6 +108,8 @@ if get_option('installed_tests')
     'org.flatpak_builder.gui.desktop',
     'org.flatpak_builder.gui.json',
     'org.flatpak_builder.gui.metainfo.xml',
+    'org.flatpak.appstream_media.json',
+    'org.test.Hello-256.png',
 
     install_dir: installed_testdir,
     install_mode: 'rw-r--r--',

tests/org.flatpak.appstream_media.json

@@ -0,0 +1,44 @@
+{
+    "id": "org.flatpak.appstream_media",
+    "runtime": "org.test.Platform",
+    "sdk": "org.test.Sdk",
+    "rename-desktop-file": "org.flatpak_builder.gui.desktop",
+    "rename-appdata-file": "org.flatpak_builder.gui.metainfo.xml",
+    "rename-icon": "org.test.Hello-256",
+    "command": "hello",
+    "modules": [
+        {
+            "name": "appstream_media",
+            "buildsystem": "simple",
+            "build-commands": [
+                "mkdir -p ${FLATPAK_DEST}/bin ${FLATPAK_DEST}/share/metainfo ${FLATPAK_DEST}/share/applications",
+                "mkdir -p ${FLATPAK_DEST}/share/icons/hicolor/256x256/apps",
+                "cp -vf hello.sh ${FLATPAK_DEST}/bin/hello",
+                "cp -vf org.flatpak_builder.gui.metainfo.xml ${FLATPAK_DEST}/share/metainfo",
+                "cp -vf org.flatpak_builder.gui.desktop ${FLATPAK_DEST}/share/applications",
+                "cp -vf org.test.Hello-256.png ${FLATPAK_DEST}/share/icons/hicolor/256x256/apps"
+            ],
+            "sources": [
+                {
+                    "type": "script",
+                    "dest-filename": "hello.sh",
+                    "commands": [
+                        "echo \"Hello world, from a sandbox\""
+                    ]
+                },
+                {
+                    "type": "file",
+                    "path": "org.flatpak_builder.gui.desktop"
+                },
+                {
+                    "type": "file",
+                    "path": "org.flatpak_builder.gui.metainfo.xml"
+                },
+                {
+                    "type": "file",
+                    "path": "org.test.Hello-256.png"
+                }
+            ]
+        }
+    ]
+}