[breaking] improved DB lookups.

Author: RillingDev

README.md

@@ -21,27 +21,30 @@ Queries the [MusicBrainz](https://musicbrainz.org/) API and fetches data from li
 
 ## Configuration
 
-This application uses Spring Boot which allows for easy handling of configurations. The following can either be passed
-as command line arguments (e.g. `--musicbrainz-enricher.host=foo`), or in a file called `application.properties` in the
-current working directory (e.g. containing `musicbrainz-enricher.host=foo`).
+This application uses Spring Boot which allows for easy handling of configurations.
+See [the Spring Boot documentation](https://docs.spring.io/spring-boot/docs/current/reference/html/howto.html#howto.properties-and-configuration.external-properties-location)
+for details.
 
 - `musicbrainz-enricher.host` (Either "test.musicbrainz.org" or "musicbrainz.org")
 - Credentials
 	- Musicbrainz
 		- `musicbrainz-enricher.musicbrainz.username`
 		- `musicbrainz-enricher.musicbrainz.password`
-	- Discogs API (can be left empty to use (slower) unauthenticated discogs API access)
+	- [Discogs API](https://www.discogs.com/developers/)(can be left empty to use (slower) unauthenticated discogs API
+	  access)
 		- `musicbrainz-enricher.discogs.token`
-	- Spotify API (can be left empty to disable spotify API access)
+	- [Spotify API](https://developer.spotify.com/documentation/web-api) (can be left empty to disable spotify API
+	  access)
 		- `musicbrainz-enricher.spotify.client-id`
 		- `musicbrainz-enricher.spotify.client-secret`
 
 ## Usage
 
-Before starting, set up <https://github.com/metabrainz/musicbrainz-docker> locally with the database port open.
+Before starting, set up a copy of the MusicBrainz database using <https://github.com/metabrainz/musicbrainz-docker>
+locally with the database port open.
 
-This tool can run in auto-query or single mode. Auto-query mode will enrich every entity from the musicbrainz database.
-Single mode takes a musicbrainz MBID and will enrich the matching entity.
+This tool can run in auto-query or single mode. Auto-query mode will enrich every entity from the MusicBrainz database.
+Single mode takes a MusicBrainz MBID and will enrich the matching entity.
 
 Auto-query mode:
 `java -jar musicbrainz-enricher*.jar 'release'`.
@@ -51,5 +54,5 @@ Single mode:
 
 ### History Storage
 
-This application will remember entities checked already and will only re-check them after duration `n` days, where `n`
-defaults to 90 days. In order to reset this, truncate the table `history_entry` in the schema `musicbrainz_enricher`.
+The application will remember entities checked already and will not re-check them. In order to reset this, truncate the
+`*_history_entry` tables in the schema `musicbrainz_enricher`.

src/main/java/dev/rilling/musicbrainzenricher/MusicbrainzEnricherService.java

@@ -31,14 +31,13 @@ public class MusicbrainzEnricherService {
 
 	public void runInAutoQueryMode(@NotNull DataType dataType) {
 		switch (dataType) {
-			case RELEASE -> musicbrainzAutoQueryService.autoQueryReleasesWithRelationships(mbid -> executeEnrichment(
+			case RELEASE -> musicbrainzAutoQueryService.autoQueryReleases(mbid -> executeEnrichment(
 				dataType,
 				mbid,
 				findFittingEnrichmentService(dataType)));
-			case RELEASE_GROUP ->
-				musicbrainzAutoQueryService.autoQueryReleaseGroupsWithRelationships(mbid -> executeEnrichment(dataType,
-					mbid,
-					findFittingEnrichmentService(dataType)));
+			case RELEASE_GROUP -> musicbrainzAutoQueryService.autoQueryReleaseGroups(mbid -> executeEnrichment(dataType,
+				mbid,
+				findFittingEnrichmentService(dataType)));
 		}
 	}
 

src/main/java/dev/rilling/musicbrainzenricher/api/musicbrainz/MusicbrainzAutoQueryService.java

@@ -28,27 +28,21 @@ public class MusicbrainzAutoQueryService {
 		this.releaseGroupRepository = releaseGroupRepository;
 	}
 
-	public void autoQueryReleasesWithRelationships(@NotNull Consumer<UUID> mbidConsumer) {
-		long count = releaseRepository.countNewReleasesWhereRelationshipsExist();
-		LOGGER.info("Found a total of {} new auto query releases.", count);
-
-		long offset = 0;
-		while (offset < count) {
-			LOGGER.info("Loading {} releases with offset {}...", LIMIT, offset);
-			releaseRepository.findNewReleaseMbidWhereRelationshipsExist(offset, LIMIT).forEach(mbidConsumer);
-			offset += LIMIT;
+	public void autoQueryReleases(@NotNull Consumer<UUID> mbidConsumer) {
+		long count = releaseRepository.countFromWorkQueue();
+		while (count > 0) {
+			LOGGER.info("{} auto query releases remaining.", count);
+			releaseRepository.findFromWorkQueue(LIMIT).forEach(mbidConsumer);
+			count = releaseRepository.countFromWorkQueue();
 		}
 	}
 
-	public void autoQueryReleaseGroupsWithRelationships(@NotNull Consumer<UUID> mbidConsumer) {
-		long count = releaseGroupRepository.countNewReleaseGroupsWhereRelationshipsExist();
-		LOGGER.info("Found a total of {} new auto query release groups.", count);
-
-		long offset = 0;
-		while (offset < count) {
-			LOGGER.info("Loading {} release groups with offset {}...", LIMIT, offset);
-			releaseGroupRepository.findNewReleaseGroupsMbidWhereRelationshipsExist(offset, LIMIT).forEach(mbidConsumer);
-			offset += LIMIT;
+	public void autoQueryReleaseGroups(@NotNull Consumer<UUID> mbidConsumer) {
+		long count = releaseGroupRepository.countFromWorkQueue();
+		while (count > 0) {
+			LOGGER.info("{} auto query release groups remaining.", count);
+			releaseGroupRepository.findFromWorkQueue(LIMIT).forEach(mbidConsumer);
+			count = releaseGroupRepository.countFromWorkQueue();
 		}
 	}
 

src/main/java/dev/rilling/musicbrainzenricher/core/ReleaseGroupRepository.java

@@ -20,28 +20,13 @@ public class ReleaseGroupRepository {
 		this.jdbcTemplate = jdbcTemplate;
 	}
 
-	public long countNewReleaseGroupsWhereRelationshipsExist() {
-		return Objects.requireNonNull(jdbcTemplate.queryForObject("""
-			SELECT COUNT(*)	FROM musicbrainz.release_group rg
-				WHERE rg.id IN
-					(SELECT lrgu.entity0 FROM musicbrainz.l_release_group_url lrgu)
-				AND rg.gid NOT IN
-				  (SELECT rghe.release_group_gid FROM musicbrainz_enricher.release_group_history_entry rghe)
-			""", Long.class));
+	public long countFromWorkQueue() {
+		return Objects.requireNonNull(jdbcTemplate.queryForObject("SELECT COUNT(*) FROM musicbrainz_enricher.release_group_work_queue", Long.class));
 	}
 
 	@NotNull
-	public List<UUID> findNewReleaseGroupsMbidWhereRelationshipsExist(long offset, int limit) {
-		List<UUID> mbids = jdbcTemplate.query("""
-			SELECT rg.gid
-			FROM musicbrainz.release_group rg
-				WHERE rg.id IN
-					  (SELECT lrgu.entity0 FROM musicbrainz.l_release_group_url	lrgu)
-				AND rg.gid NOT IN
-					(SELECT rghe.release_group_gid FROM musicbrainz_enricher.release_group_history_entry rghe)
-			ORDER BY rg.id
-			OFFSET ? LIMIT ?
-			""", (rs, rowNum) -> rs.getObject("gid", UUID.class), offset, limit);
+	public List<UUID> findFromWorkQueue(long offset) {
+		List<UUID> mbids = jdbcTemplate.query("SELECT rg.gid FROM musicbrainz_enricher.release_group_work_queue rg LIMIT ?", (rs, rowNum) -> rs.getObject("gid", UUID.class), offset);
 		return Collections.unmodifiableList(mbids);
 	}
 }

src/main/java/dev/rilling/musicbrainzenricher/core/ReleaseRepository.java

@@ -19,27 +19,13 @@ public class ReleaseRepository {
 		this.jdbcTemplate = jdbcTemplate;
 	}
 
-	public long countNewReleasesWhereRelationshipsExist() {
-		return Objects.requireNonNull(jdbcTemplate.queryForObject("""
-			SELECT COUNT(*) FROM musicbrainz.release r
-				WHERE r.id IN
-					(SELECT lru.entity0 FROM musicbrainz.l_release_url lru)
-				AND r.gid NOT IN
-				  (SELECT rhe.release_gid FROM musicbrainz_enricher.release_history_entry rhe)
-			""", Long.class));
+	public long countFromWorkQueue() {
+		return Objects.requireNonNull(jdbcTemplate.queryForObject("SELECT COUNT(*) FROM musicbrainz_enricher.release_work_queue", Long.class));
 	}
 
 	@NotNull
-	public List<UUID> findNewReleaseMbidWhereRelationshipsExist(long offset, int limit) {
-		List<UUID> mbids = jdbcTemplate.query("""
-			SELECT r.gid FROM musicbrainz.release r
-				WHERE r.id IN
-					(SELECT lru.entity0 FROM musicbrainz.l_release_url lru)
-				AND r.gid NOT IN
-				  (SELECT rhe.release_gid FROM musicbrainz_enricher.release_history_entry rhe)
-			ORDER BY r.id
-			OFFSET ? LIMIT ?
-			""", (rs, rowNum) -> rs.getObject("gid", UUID.class), offset, limit);
+	public List<UUID> findFromWorkQueue(int limit) {
+		List<UUID> mbids = jdbcTemplate.query("SELECT r.gid FROM musicbrainz_enricher.release_work_queue r LIMIT ?", (rs, rowNum) -> rs.getObject("gid", UUID.class), limit);
 		return Collections.unmodifiableList(mbids);
 	}
 }

src/main/resources/db/migration/V1_0__create_history_schema.sql

@@ -4,8 +4,31 @@ CREATE TABLE musicbrainz_enricher.release_history_entry
 		ON DELETE CASCADE
 );
 
+CREATE MATERIALIZED VIEW musicbrainz_enricher.release_with_relationships AS
+SELECT *
+FROM musicbrainz.release r
+WHERE r.id IN
+	  (SELECT lru.entity0 FROM musicbrainz.l_release_url lru);
+
+CREATE VIEW musicbrainz_enricher.release_work_queue AS
+SELECT *
+FROM musicbrainz_enricher.release_with_relationships r
+WHERE r.gid NOT IN (SELECT rhe.release_gid FROM musicbrainz_enricher.release_history_entry rhe);
+
+
 CREATE TABLE musicbrainz_enricher.release_group_history_entry
 (
 	release_group_gid UUID PRIMARY KEY NOT NULL REFERENCES musicbrainz.release_group (gid)
 		ON DELETE CASCADE
 );
+
+CREATE MATERIALIZED VIEW musicbrainz_enricher.release_group_with_relationships AS
+SELECT *
+FROM musicbrainz.release_group rg
+WHERE rg.id IN
+	  (SELECT lrgu.entity0 FROM musicbrainz.l_release_group_url lrgu);
+
+CREATE VIEW musicbrainz_enricher.release_group_work_queue AS
+SELECT *
+FROM musicbrainz_enricher.release_group_with_relationships rg
+WHERE rg.gid NOT IN (SELECT rghe.release_group_gid FROM musicbrainz_enricher.release_group_history_entry rghe);