Support the new swcatalog catalog metadata location and add app-info …

…fallback

contrib/apt-conf/50appstream

@@ -73,5 +73,5 @@ Acquire::IndexTargets {
 
 # Refresh AppStream cache when APT's cache is updated (i.e. apt update)
 APT::Update::Post-Invoke-Success {
-    "if /usr/bin/test -w /var/cache/app-info -a -e /usr/bin/appstreamcli; then appstreamcli refresh-cache > /dev/null || true; fi";
+    "if /usr/bin/test -w /var/cache/swcatalog -a -e /usr/bin/appstreamcli; then appstreamcli refresh-cache > /dev/null || true; fi";
 };

docs/xml/collection-iconcache.xml

@@ -20,19 +20,19 @@
 	<section id="spec-iconcache-location">
 		<title>Filesystem locations</title>
 		<para>
-			All icons of type <literal>cached</literal> must be placed in <filename>/usr/share/app-info/icons/%{origin}/%{size}/</filename> or <filename>/var/cache/app-info/icons/%{origin}/%{size}/</filename>,
+			All icons of type <literal>cached</literal> must be placed in <filename>/usr/share/swcatalog/icons/%{origin}/%{size}/</filename> or <filename>/var/cache/swcatalog/icons/%{origin}/%{size}/</filename>,
 			where <literal>origin</literal> is the AppStream data origin defined in the AppStream data file (see <xref linkend="spec-asxml-general"/>), and <literal>size</literal>
 			is <code>64x64</code> or <code>128x128</code> depending on the size of the icon. And icon might be present with different sizes in both directories.
 		</para>
 		<para>
 			For example the cache icon <code>krita.png</code> of a component in a data file with the origin <code>jessie</code> should be stored in
-			<filename>/usr/share/app-info/icons/jessie/64x64/krita.png</filename> (or in the <filename>/var/cache</filename> location).
+			<filename>/usr/share/swcatalog/icons/jessie/64x64/krita.png</filename> (or in the <filename>/var/cache</filename> location).
 		</para>
 		<para>
 			Icon scaling factors commonly used for HiDPI display support are part of the size-directory filename and are separated from the regular size via an <code>@</code> sign.
 			If the scaling factor is 1, it must be omitted from the directory name.
 			For example, if the icon scaling factor is <literal>2</literal> for icons of size <literal>64x64</literal> from origin <literal>jessie</literal>, the icon must be placed
-			in <filename>/usr/share/app-info/icons/jessie/64x64@2/</filename>.
+			in <filename>/usr/share/swcatalog/icons/jessie/64x64@2/</filename>.
 		</para>
 		<note>
 			<title>Legacy Support</title>

docs/xml/collection-xmldata.xml

@@ -26,12 +26,21 @@
 			3rd-party repositories use a vendor name and repository-name combination, for example Ubuntu PPAs might get <filename>ppa-ubuntu12.04-username-foobar.xml</filename>.
 		</para>
 		<para>
-			There are two valid locations to store AppStream XML data. <filename>/usr/share/app-info/xml</filename> stores all AppStream data which
-			has been installed via software packages, while <filename>/var/cache/app-info/xml</filename> stores application data which was downloaded
+			There are two valid locations to store AppStream XML data. <filename>/usr/share/swcatalog/xml</filename> stores all AppStream data which
+			has been installed via software packages, while <filename>/var/lib/swcatalog/xml</filename> stores application data which was downloaded
 			by the package manager or placed there by other tools (for example, Limba).
 			The XML files can either be plain files or be compressed with gzip. It is always a good idea to compress the files, because they tend to become
 			quite large.
 		</para>
+
+		<important>
+			<title>Legacy Path</title>
+			<para>
+				AppStream tools scan the paths <filename>/usr/share/app-info/(xml|xmls)</filename> / <filename>/var/lib/app-info/(xml|xmls)</filename> path for legacy
+				compatibility as well. If possible, the old locations and old layouts should not be used anymore.
+				Support for the legacy path will likely be dropped completely with a future AppStream 1.0 release.
+			</para>
+		</important>
 	</section>
 
 	<section id="spec-asxml-general">

docs/xml/collection-yamldata.xml

@@ -24,10 +24,6 @@
 			</para>
 		</important>
 
-		<para>
-			The DEP-11 YAML metadata can be validated for correctness using the <command>dep11-validate</command> tool from the
-			<ulink url="https://packages.debian.org/source/sid/appstream-dep11">AppStream DEP-11 utils</ulink>.
-		</para>
 		<para>
 			Fields not mentioned in this document are not recognized by DEP-11 YAML parsers.
 		</para>
@@ -37,9 +33,9 @@
 	<section id="spec-dep11-filenaming">
 		<title>File naming and location</title>
 		<para>
-			Take a look at <xref linkend="spec-asxml-filenaming"/> for AppStream XML files. While the XML data belongs into the <filename>xmls</filename> subdirectory in
-			<filename>/usr/share/app-info</filename> (or <filename>/var/cache/app-info</filename>), the YAML data is stored in the <filename>yaml</filename> subdirectory.
-			All other rules affecting the XML apply the DEP-11 YAML as well, including the recommendation to compress the files with gzip.
+			Take a look at <xref linkend="spec-asxml-filenaming"/> for AppStream XML files. While the XML data belongs into the <filename>xml</filename> subdirectory in
+			<filename>/usr/share/swcatalog</filename> (or <filename>/var/cache/swcatalog</filename>), the YAML data is stored in the <filename>yaml</filename> subdirectory.
+			All other rules affecting the XML apply the DEP-11 YAML as well, including the recommendation to compress the files with gzip and the icon search logic.
 		</para>
 	</section>
 

src/as-cache.c

@@ -66,6 +66,9 @@ typedef struct
 G_DEFINE_TYPE_WITH_PRIVATE (AsCache, as_cache, G_TYPE_OBJECT)
 #define GET_PRIVATE(o) (as_cache_get_instance_private (o))
 
+/* location of the system-wide cache */
+static const gchar *AS_APPSTREAM_SYS_CACHE_DIR = "/var/cache/swcatalog/cache";
+
 typedef struct {
 	gboolean		is_os_data;
 	gboolean		is_mask;

src/as-distro-extras.c

@@ -45,8 +45,10 @@
 #define YAML_SEPARATOR_LEN strlen(YAML_SEPARATOR)
 
 static const gchar *apt_lists_dir = "/var/lib/apt/lists/";
-static const gchar *appstream_yml_target = "/var/lib/app-info/yaml";
-static const gchar *appstream_icons_target = "/var/lib/app-info/icons";
+static const gchar *appstream_yaml_target = "/var/lib/swcatalog/yaml";
+static const gchar *appstream_icons_target = "/var/lib/swcatalog/icons";
+static const gchar *appstream_catalog_root = "/var/lib/swcatalog";
+static const gchar *appstream_catalog_legacy_root = "/var/lib/app-info";
 
 static const gchar* const default_icon_sizes[] = { "48x48", "48x48@2", "64x64", "64x64@2", "128x128", "128x128@2", NULL };
 
@@ -222,7 +224,7 @@ as_pool_scan_apt (AsPool *pool, gboolean force, GError **error)
 	g_autoptr(GError) tmp_error = NULL;
 	gboolean data_changed = FALSE;
 	gboolean icons_available = FALSE;
-	guint i;
+	gboolean yaml_target_dir_exists = FALSE;
 
 	g_debug ("Scanning for metadata changes in the APT cache.");
 
@@ -232,23 +234,24 @@ as_pool_scan_apt (AsPool *pool, gboolean force, GError **error)
 		return;
 	}
 
-	if (g_file_test (appstream_yml_target, G_FILE_TEST_IS_DIR)) {
+	yaml_target_dir_exists = g_file_test (appstream_yaml_target, G_FILE_TEST_IS_DIR);
+	if (yaml_target_dir_exists) {
 		g_autoptr(GPtrArray) ytfiles = NULL;
 
 		/* we can't modify the files here if we don't have write access */
-		if (!as_utils_is_writable (appstream_yml_target)) {
-			g_debug ("Unable to write to '%s': Can't add AppStream data from APT to the pool.", appstream_yml_target);
+		if (!as_utils_is_writable (appstream_yaml_target)) {
+			g_debug ("Unable to write to '%s': Can't add AppStream data from APT to the pool.", appstream_yaml_target);
 			return;
 		}
 
-		ytfiles = as_utils_find_files_matching (appstream_yml_target, "*", FALSE, &tmp_error);
+		ytfiles = as_utils_find_files_matching (appstream_yaml_target, "*", FALSE, &tmp_error);
 		if (tmp_error != NULL) {
 			g_warning ("Could not scan for broken symlinks in DEP-11 target: %s", tmp_error->message);
 			return;
 		}
 
 		if (ytfiles != NULL) {
-			for (i = 0; i < ytfiles->len; i++) {
+			for (guint i = 0; i < ytfiles->len; i++) {
 				const gchar *fname = (const gchar*) g_ptr_array_index (ytfiles, i);
 				if (!g_file_test (fname, G_FILE_TEST_EXISTS)) {
 					g_remove (fname);
@@ -277,13 +280,13 @@ as_pool_scan_apt (AsPool *pool, gboolean force, GError **error)
 	 *
 	 * We also check for available icons, to install icons again if they were disabled (and removed) previously.
 	 */
-	for (i = 0; i < yml_files->len; i++) {
+	for (guint i = 0; i < yml_files->len; i++) {
 		g_autofree gchar *fbasename = NULL;
 		g_autofree gchar *dest_fname = NULL;
 		const gchar *fname = (const gchar*) g_ptr_array_index (yml_files, i);
 
 		fbasename = g_path_get_basename (fname);
-		dest_fname = g_build_filename (appstream_yml_target, fbasename, NULL);
+		dest_fname = g_build_filename (appstream_yaml_target, fbasename, NULL);
 		if (!g_file_test (dest_fname, G_FILE_TEST_EXISTS)) {
 			data_changed = TRUE;
 			g_debug ("File '%s' missing, cache update is needed.", dest_fname);
@@ -332,15 +335,27 @@ as_pool_scan_apt (AsPool *pool, gboolean force, GError **error)
 	 * So, we hereby simply "own" the icons directory and all it's contents, anything put in there by 3rd-parties will
 	 * be deleted.
 	 * (And there should actually be no cases 3rd-parties put icons there on a Debian machine, since metadata in packages
-	 * will land in /usr/share/app-info anyway)
+	 * will land in /usr/share/swcatalog anyway)
 	 */
 	as_utils_delete_dir_recursive (appstream_icons_target);
-	if (g_mkdir_with_parents (appstream_yml_target, 0755) > 0) {
-		g_debug ("Unable to create '%s': %s", appstream_yml_target, g_strerror (errno));
-		return;
+
+	if (!yaml_target_dir_exists) {
+
+		/* create YAML target directory */
+		if (g_mkdir_with_parents (appstream_yaml_target, 0755) > 0) {
+			g_warning ("Unable to create '%s': %s", appstream_yaml_target, g_strerror (errno));
+			return;
+		}
+
+		/* create compatibility symlink for old location */
+		if (!g_file_test (appstream_catalog_legacy_root, G_FILE_TEST_EXISTS)) {
+			if (symlink (appstream_catalog_root, appstream_catalog_legacy_root) != 0)
+				g_debug ("Unable to create compatibility symlink '%s': %s",
+					 appstream_catalog_legacy_root, g_strerror (errno));
+		}
 	}
 
-	for (i = 0; i < yml_files->len; i++) {
+	for (guint i = 0; i < yml_files->len; i++) {
 		g_autofree gchar *fbasename = NULL;
 		g_autofree gchar *dest_fname = NULL;
 		g_autofree gchar *origin = NULL;
@@ -349,7 +364,7 @@ as_pool_scan_apt (AsPool *pool, gboolean force, GError **error)
 		const gchar *fname = (const gchar*) g_ptr_array_index (yml_files, i);
 
 		fbasename = g_path_get_basename (fname);
-		dest_fname = g_build_filename (appstream_yml_target, fbasename, NULL);
+		dest_fname = g_build_filename (appstream_yaml_target, fbasename, NULL);
 
 		if (!g_file_test (fname, G_FILE_TEST_EXISTS)) {
 			/* broken symlinks in the dest will have been removed earlier */
@@ -391,5 +406,5 @@ as_pool_scan_apt (AsPool *pool, gboolean force, GError **error)
 	}
 
 	/* ensure the cache-rebuild process notices these changes */
-	as_touch_location (appstream_yml_target);
+	as_touch_location (appstream_yaml_target);
 }

src/as-pool.c

@@ -102,10 +102,12 @@ static guint signals [SIGNAL_LAST] = { 0 };
  */
 #define AS_SEARCH_GREYLIST_STR _("app;application;package;program;programme;suite;tool")
 
-/* Locations where system-wide AppStream collection metadata may be stored. */
-static const gchar *SYSTEM_COLLECTION_METADATA_PATHS[] = { "/usr/share/app-info",
-							   "/var/lib/app-info",
-							   "/var/cache/app-info",
+/* Prefixes of locations where system-wide AppStream catalog metadata can be found.
+ * TODO: We should really parse $XDG_DATA_DIRS for the /usr location in a safe way,
+ * instead of hardcoding one canonical path here. */
+static const gchar *SYSTEM_CATALOG_METADATA_PREFIXES[] = { "/usr/share",
+							   "/var/lib",
+							   "/var/cache",
 							   NULL};
 
 /* where .desktop files are installed to by packages to be registered with the system */
@@ -441,26 +443,28 @@ as_pool_class_init (AsPoolClass *klass)
 }
 
 /**
- * as_pool_add_collection_metadata_dir_internal:
+ * as_pool_add_catalog_metadata_dir_internal:
  * @pool: An instance of #AsPool.
  * @lgroup: The location grouping to add to.
  * @directory: An existing filesystem location.
  * @add_root: Whether to add the root directory if necessary.
+ * @with_legacy_support: Whether some legacy support should be enabled.
  *
  * See %as_pool_add_metadata_location()
  */
 static void
-as_pool_add_collection_metadata_dir_internal (AsPool *pool,
-					      AsLocationGroup *lgroup,
-					      const gchar *directory,
-					      gboolean add_root)
+as_pool_add_catalog_metadata_dir_internal (AsPool *pool,
+					   AsLocationGroup *lgroup,
+					   const gchar *directory,
+					   gboolean add_root,
+					   gboolean with_legacy_support)
 {
 	gboolean dir_added = FALSE;
 	g_autofree gchar *icon_dir = NULL;
 	gchar *path;
 
 	if (!g_file_test (directory, G_FILE_TEST_IS_DIR)) {
-		g_debug ("Not adding metadata location '%s': Not a directory, or does not exist.",
+		g_debug ("Not adding metadata catalog location '%s': Not a directory, or does not exist.",
 			 directory);
 		return;
 	}
@@ -481,15 +485,17 @@ as_pool_add_collection_metadata_dir_internal (AsPool *pool,
 	}
 	g_free (path);
 
-	path = g_build_filename (directory, "xmls", NULL);
-	if (g_file_test (path, G_FILE_TEST_IS_DIR)) {
-		as_location_group_add_dir (lgroup,
-					   path,
-					   icon_dir,
-					   AS_FORMAT_KIND_XML);
-		dir_added = TRUE;
+	if (with_legacy_support) {
+		path = g_build_filename (directory, "xmls", NULL);
+		if (g_file_test (path, G_FILE_TEST_IS_DIR)) {
+			as_location_group_add_dir (lgroup,
+						path,
+						icon_dir,
+						AS_FORMAT_KIND_XML);
+			dir_added = TRUE;
+		}
+		g_free (path);
 	}
-	g_free (path);
 
 	path = g_build_filename (directory, "yaml", NULL);
 	if (g_file_test (path, G_FILE_TEST_IS_DIR)) {
@@ -637,7 +643,7 @@ static void
 as_pool_detect_std_metadata_dirs (AsPool *pool, gboolean include_user_data)
 {
 	AsPoolPrivate *priv = GET_PRIVATE (pool);
-	AsLocationGroup *lgroup_coll;
+	AsLocationGroup *lgroup_catalog;
 	AsLocationGroup *lgroup_metainfo;
 
 	/* NOTE: Data must be write-locked by the calling function. */
@@ -646,14 +652,14 @@ as_pool_detect_std_metadata_dirs (AsPool *pool, gboolean include_user_data)
 	g_hash_table_remove_all (priv->std_data_locations);
 
 	/* create location groups and register them */
-	lgroup_coll = as_location_group_new (pool,
+	lgroup_catalog = as_location_group_new (pool,
 					     AS_COMPONENT_SCOPE_SYSTEM,
 					     AS_FORMAT_STYLE_COLLECTION,
 					     TRUE, /* is OS data */
 					     OS_COLLECTION_CACHE_KEY);
 	g_hash_table_insert (priv->std_data_locations,
-			     g_strdup (lgroup_coll->cache_key),
-			     lgroup_coll);
+			     g_strdup (lgroup_catalog->cache_key),
+			     lgroup_catalog);
 	lgroup_metainfo = as_location_group_new (pool,
 						 AS_COMPONENT_SCOPE_SYSTEM,
 						 AS_FORMAT_STYLE_METAINFO,
@@ -691,11 +697,37 @@ as_pool_detect_std_metadata_dirs (AsPool *pool, gboolean include_user_data)
 	/* add collection XML directories for the OS */
 	/* check if we are permitted to load this */
 	if (as_flags_contains (priv->flags, AS_POOL_FLAG_LOAD_OS_COLLECTION)) {
-		for (guint i = 0; SYSTEM_COLLECTION_METADATA_PATHS[i] != NULL; i++) {
-			as_pool_add_collection_metadata_dir_internal (pool,
-								lgroup_coll,
-								SYSTEM_COLLECTION_METADATA_PATHS[i],
-								FALSE);
+		for (guint i = 0; SYSTEM_CATALOG_METADATA_PREFIXES[i] != NULL; i++) {
+			g_autofree gchar *catalog_path = NULL;
+			g_autofree gchar *catalog_legacy_path = NULL;
+			gboolean ignore_legacy_path = FALSE;
+
+			catalog_path = g_build_filename (SYSTEM_CATALOG_METADATA_PREFIXES[i], "swcatalog", NULL);
+			catalog_legacy_path = g_build_filename (SYSTEM_CATALOG_METADATA_PREFIXES[i], "app-info", NULL);
+
+			/* ignore compatibility symlink if one exists */
+			if (g_file_test (catalog_legacy_path, G_FILE_TEST_IS_SYMLINK)) {
+				g_autofree gchar *link_target = g_file_read_link (catalog_legacy_path, NULL);
+				if (link_target != NULL) {
+					if (g_strcmp0 (link_target, catalog_path) == 0) {
+						ignore_legacy_path = TRUE;
+						g_debug ("Ignoring legacy catalog location '%s'.", catalog_legacy_path);
+					}
+				}
+			}
+
+			as_pool_add_catalog_metadata_dir_internal (pool,
+								   lgroup_catalog,
+								   catalog_path,
+								   FALSE, /* add root */
+								   FALSE /* no legacy support */);
+
+			if (!ignore_legacy_path)
+				as_pool_add_catalog_metadata_dir_internal (pool,
+									   lgroup_catalog,
+									   catalog_legacy_path,
+									   FALSE, /* add root */
+									   TRUE /* enable legacy support */);
 		}
 	}
 
@@ -2347,10 +2379,11 @@ as_pool_add_extra_data_location (AsPool *pool, const gchar *directory, AsFormatS
 	g_hash_table_insert (priv->extra_data_locations,
 			     g_strdup (extra_group->cache_key),
 			     extra_group);
-	as_pool_add_collection_metadata_dir_internal (pool,
-						      extra_group,
-						      directory,
-						      TRUE);
+	as_pool_add_catalog_metadata_dir_internal (pool,
+						   extra_group,
+						   directory,
+						   TRUE /* add root */,
+						   FALSE /* no legacy support */);
 }
 
 /**

src/as-settings-private.h

@@ -29,7 +29,6 @@ G_BEGIN_DECLS
 #define AS_INTERNAL_VISIBLE __attribute__((visibility("default")))
 
 #define AS_CONFIG_NAME "/etc/appstream.conf"
-#define AS_APPSTREAM_SYS_CACHE_DIR "/var/cache/app-info/cache"
 
 #pragma GCC visibility pop
 G_END_DECLS

src/as-utils.c

@@ -2046,7 +2046,7 @@ as_utils_install_icon_tarball (AsMetadataLocation location,
 				GError **error)
 {
 	g_autofree gchar *dir = NULL;
-	dir = g_strdup_printf ("%s%s/app-info/icons/%s/%s",
+	dir = g_strdup_printf ("%s%s/swcatalog/icons/%s/%s",
 			       destdir,
 			       as_metadata_location_get_prefix (location),
 			       origin,
@@ -2111,11 +2111,11 @@ as_utils_install_metadata_file (AsMetadataLocation location,
 	case AS_FORMAT_STYLE_COLLECTION:
 		if (g_strstr_len (filename, -1, ".yml.gz") != NULL) {
 			path = g_build_filename (as_metadata_location_get_prefix (location),
-						 "app-info", "yaml", NULL);
+						 "swcatalog", "yaml", NULL);
 			ret = as_utils_install_metadata_file_internal (filename, origin, path, destdir, TRUE, error);
 		} else {
 			path = g_build_filename (as_metadata_location_get_prefix (location),
-						 "app-info", "xml", NULL);
+						 "swcatalog", "xml", NULL);
 			ret = as_utils_install_metadata_file_internal (filename, origin, path, destdir, FALSE, error);
 		}
 		break;