No point building sitemaps for entities blocked in robots.txt

metabrainz · Oct 8, 2015 · b37cf66 · b37cf66
1 parent 0325630
commit b37cf66
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/lib/MusicBrainz/Server/Sitemap/Incremental.pm b/lib/MusicBrainz/Server/Sitemap/Incremental.pm
@@ -68,7 +68,9 @@ This script works by:
 
 =cut
 
-my @LASTMOD_ENTITIES = entities_with('sitemaps_lastmod_table');
+my %INDEXABLE_ENTITIES = map { $_ => 1 } entities_with(['mbid', 'indexable']);
+my @LASTMOD_ENTITIES = grep { exists $INDEXABLE_ENTITIES{$_} }
+                       entities_with('sitemaps_lastmod_table');
 my %LASTMOD_ENTITIES = map { $_ => 1 } @LASTMOD_ENTITIES;
 
 memoize('get_primary_keys');