fix(filters): RED and OPS lossless parsing and filtering (#1373)

* fix(filters): RED and OPS lossless parsing and filtering

* fix(filters): logscore and EP parsing

* fix(filters): tests

* fix(filters): tests

* feat(definitions): RED parse title variable

* feat(indexers): setup indexer to filter tests

* feat(indexers): tests and improve parsing

* feat(indexers): improve tests

internal/domain/filter.go

@@ -475,7 +475,7 @@ func (f *Filter) CheckFilter(r *Release) ([]string, bool) {
 		f.addRejectionF("formats not matching. got: %v want: %v", r.Audio, f.Formats)
 	}
 
-	if len(f.Quality) > 0 && !sliceContainsSlice(r.Audio, f.Quality) {
+	if len(f.Quality) > 0 && !containsMatchBasic(r.Audio, f.Quality) {
 		f.addRejectionF("quality not matching. got: %v want: %v", r.Audio, f.Quality)
 	}
 
@@ -547,7 +547,7 @@ func (f *Filter) CheckFilter(r *Release) ([]string, bool) {
 	return nil, true
 }
 
-func (f Filter) checkMaxDownloads() bool {
+func (f *Filter) checkMaxDownloads() bool {
 	if f.Downloads == nil {
 		return false
 	}
@@ -580,7 +580,7 @@ func (f *Filter) isPerfectFLAC(r *Release) bool {
 	if !containsAny(r.Audio, "Log") {
 		return false
 	}
-	if !containsAny(r.Audio, "Log100") {
+	if !containsAny(r.Audio, "Log100") || r.LogScore != 100 {
 		return false
 	}
 	if !containsAny(r.Audio, "FLAC") {
@@ -616,6 +616,32 @@ func (f *Filter) checkSizeFilter(r *Release) bool {
 	return true
 }
 
+// IsPerfectFLAC Perfect is "CD FLAC Cue Log 100% Lossless or 24bit Lossless"
+func (f *Filter) IsPerfectFLAC(r *Release) ([]string, bool) {
+	rejections := []string{}
+
+	if r.Source != "CD" {
+		rejections = append(rejections, fmt.Sprintf("wanted Source CD, got %s", r.Source))
+	}
+	if r.AudioFormat != "FLAC" {
+		rejections = append(rejections, fmt.Sprintf("wanted Format FLAC, got %s", r.AudioFormat))
+	}
+	if !r.HasCue {
+		rejections = append(rejections, fmt.Sprintf("wanted Cue, got %t", r.HasCue))
+	}
+	if !r.HasLog {
+		rejections = append(rejections, fmt.Sprintf("wanted Log, got %t", r.HasLog))
+	}
+	if r.LogScore != 100 {
+		rejections = append(rejections, fmt.Sprintf("wanted Log Score 100, got %d", r.LogScore))
+	}
+	if !containsSlice(r.Bitrate, []string{"Lossless", "24bit Lossless"}) {
+		rejections = append(rejections, fmt.Sprintf("wanted Bitrate Lossless / 24bit Lossless, got %s", r.Bitrate))
+	}
+
+	return rejections, len(rejections) == 0
+}
+
 func (f *Filter) addRejection(reason string) {
 	f.Rejections = append(f.Rejections, reason)
 }

internal/domain/filter_test.go

@@ -1165,16 +1165,17 @@ func TestFilter_CheckFilter(t *testing.T) {
 					LogScore: 100,
 					Cue:      true,
 				},
-				rejections: []string{"quality not matching. got: [FLAC Lossless Log100 Log] want: [24bit Lossless]", "wanted: cue", "log score. got: 0 want: 100"},
+				rejections: []string{"quality not matching. got: [FLAC Lossless Log100 Log] want: [24bit Lossless]", "wanted: cue"},
 			},
 			want: false,
 		},
 		{
 			name: "match_music_5",
 			fields: &Release{
-				TorrentName: "Artist - Albumname FLAC CD",
+				//TorrentName: "Artist - Albumname FLAC CD",
+				TorrentName: "Artist - Albumname [2022] [Album] (FLAC 24bit Lossless CD)",
 				Year:        2022,
-				ReleaseTags: "FLAC / Lossless / Log / 100% / Cue / CD",
+				ReleaseTags: "FLAC / 24bit Lossless / Log / 100% / Cue / CD",
 				Category:    "Album",
 			},
 			args: args{
@@ -1185,11 +1186,12 @@ func TestFilter_CheckFilter(t *testing.T) {
 					Artists:           "Artist",
 					Media:             []string{"CD"},
 					Formats:           []string{"FLAC"},
-					Quality:           []string{"24bit Lossless", "Lossless"},
-					PerfectFlac:       true,
-					Log:               true,
+					Quality:           []string{"24bit Lossless"},
+					//PerfectFlac:       true,
+					//Log:               true,
 					//LogScore:          100,
 					Cue: true,
+					//Cue: true,
 				},
 			},
 			want: true,
@@ -1214,7 +1216,7 @@ func TestFilter_CheckFilter(t *testing.T) {
 					LogScore:          100,
 					Cue:               true,
 				},
-				rejections: []string{"release type not matching. got: Album want: [Single]", "log score. got: 0 want: 100"},
+				rejections: []string{"release type not matching. got: Album want: [Single]"},
 			},
 			want: false,
 		},
@@ -1238,7 +1240,7 @@ func TestFilter_CheckFilter(t *testing.T) {
 					LogScore:          100,
 					Cue:               true,
 				},
-				rejections: []string{"artists not matching. got: Artist want: Artiiiist", "log score. got: 0 want: 100"},
+				rejections: []string{"artists not matching. got: Artist want: Artiiiist"},
 			},
 			want: false,
 		},
@@ -1266,6 +1268,28 @@ func TestFilter_CheckFilter(t *testing.T) {
 			},
 			want: true,
 		},
+		{
+			name: "match_music_9",
+			fields: &Release{
+				TorrentName: "Artist - Albumname [2022] [Album] (FLAC 24bit Lossless CD)",
+				Year:        2022,
+				ReleaseTags: "FLAC / 24bit Lossless / Log / 100% / Cue / CD",
+				Category:    "Album",
+			},
+			args: args{
+				filter: Filter{
+					Enabled:           true,
+					MatchReleaseTypes: []string{"Album"},
+					Years:             "2020-2022",
+					Artists:           "Artist",
+					Media:             []string{"CD"},
+					Formats:           []string{"FLAC"},
+					Quality:           []string{"Lossless"},
+				},
+				rejections: []string{"quality not matching. got: [24BIT Lossless Cue FLAC Log100 Log] want: [Lossless]"},
+			},
+			want: false,
+		},
 		{
 			name: "match_anime_1",
 			fields: &Release{
@@ -2130,8 +2154,8 @@ func Test_matchRegex(t *testing.T) {
 		{name: "test_5", args: args{tag: "Some.show.S01.DV.2160p.ATVP.WEB-DL.DDPA5.1.x265-GROUP2", filter: ".*1080p.+(group1|group3),.*720p.+,"}, want: false},
 		{name: "test_6", args: args{tag: "[Group] -Name of a Novel Something Good-  [2012][Translated (Group)][EPUB]", filter: "(?:.*Something Good.*|.*Something Bad.*)"}, want: true},
 		{name: "test_7", args: args{tag: "[Group] -Name of a Novel Something Good-  [2012][Translated (Group)][EPUB]", filter: "(?:.*Something Funny.*|.*Something Bad.*)"}, want: false},
-		{name: "test_8", args: args{tag: ".s10E123.", filter:`\.[Ss]\d{1,2}[Ee]\d{1,3}\.`}, want: true},
-		{name: "test_9", args: args{tag: "S1E1", filter:`\.[Ss]\d{1,2}[Ee]\d{1,3}\.`}, want: false},
+		{name: "test_8", args: args{tag: ".s10E123.", filter: `\.[Ss]\d{1,2}[Ee]\d{1,3}\.`}, want: true},
+		{name: "test_9", args: args{tag: "S1E1", filter: `\.[Ss]\d{1,2}[Ee]\d{1,3}\.`}, want: false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/domain/indexer.go

@@ -352,6 +352,8 @@ func (p *IndexerIRCParse) Parse(def *IndexerDefinition, vars map[string]string,
 		parser = IRCParserGazelleGames{}
 	case "ops":
 		parser = IRCParserOrpheus{}
+	case "redacted":
+		parser = IRCParserRedacted{}
 	default:
 		parser = IRCParserDefault{}
 	}

internal/domain/indexer_test.go

@@ -335,11 +335,14 @@ func TestIRCParserOrpheus_Parse(t *testing.T) {
 				rls: NewRelease("ops"),
 				vars: map[string]string{
 					"torrentName": "Busta Rhymes – BEACH BALL (feat. BIA) – [2023] [Single] WEB/FLAC/24bit Lossless",
+					"title":       "Busta Rhymes – BEACH BALL (feat. BIA)",
+					"year":        "2023",
+					"releaseTags": "WEB/FLAC/24bit Lossless",
 				},
 			},
 			want: want{
-				title:   "BEACH BALL",
-				release: "Busta Rhymes - BEACH BALL (feat. BIA) - [2023] [Single] WEB/FLAC/24bit Lossless",
+				title:   "Busta Rhymes - BEACH BALL (feat. BIA)",
+				release: "Busta Rhymes - BEACH BALL (feat. BIA) [2023] (WEB FLAC 24BIT Lossless)",
 			},
 		},
 		{
@@ -348,11 +351,14 @@ func TestIRCParserOrpheus_Parse(t *testing.T) {
 				rls: NewRelease("ops"),
 				vars: map[string]string{
 					"torrentName": "Busta Rhymes – BEACH BALL (feat. BIA) – [2023] [Single] CD/FLAC/Lossless",
+					"title":       "Busta Rhymes – BEACH BALL (feat. BIA)",
+					"year":        "2023",
+					"releaseTags": "CD/FLAC/Lossless",
 				},
 			},
 			want: want{
-				title:   "BEACH BALL",
-				release: "Busta Rhymes - BEACH BALL (feat. BIA) - [2023] [Single] CD/FLAC/Lossless",
+				title:   "Busta Rhymes - BEACH BALL (feat. BIA)",
+				release: "Busta Rhymes - BEACH BALL (feat. BIA) [2023] (CD FLAC Lossless)",
 			},
 		},
 	}

internal/domain/irc.go

@@ -6,6 +6,7 @@ package domain
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"strings"
 	"time"
 )
@@ -185,13 +186,88 @@ func (p IRCParserGazelleGames) Parse(rls *Release, vars map[string]string) error
 
 type IRCParserOrpheus struct{}
 
+func (p IRCParserOrpheus) replaceSeparator(s string) string {
+	return strings.ReplaceAll(s, "–", "-")
+}
+
 func (p IRCParserOrpheus) Parse(rls *Release, vars map[string]string) error {
 	// OPS uses en-dashes as separators, which causes moistari/rls to not parse the torrentName properly,
 	// we replace the en-dashes with hyphens here
-	torrentName := vars["torrentName"]
-	rls.TorrentName = strings.ReplaceAll(torrentName, "–", "-")
+	torrentName := p.replaceSeparator(vars["torrentName"])
+	title := p.replaceSeparator(vars["title"])
 
-	rls.ParseString(rls.TorrentName)
+	year := vars["year"]
+	releaseTagsString := vars["releaseTags"]
+
+	//cleanTags := strings.ReplaceAll(releaseTagsString, "/", " ")
+	cleanTags := CleanReleaseTags(releaseTagsString)
+
+	tags := ParseReleaseTagString(cleanTags)
+	rls.ReleaseTags = cleanTags
+
+	audio := []string{}
+	if tags.Source != "" {
+		audio = append(audio, tags.Source)
+	}
+	if tags.AudioFormat != "" {
+		audio = append(audio, tags.AudioFormat)
+	}
+	if tags.AudioBitrate != "" {
+		audio = append(audio, tags.AudioBitrate)
+	}
+	rls.Bitrate = tags.AudioBitrate
+	rls.AudioFormat = tags.AudioFormat
+
+	// set log score
+	rls.HasLog = tags.HasLog
+	rls.LogScore = tags.LogScore
+	rls.HasCue = tags.HasCue
+
+	// Construct new release name so we have full control. We remove category such as EP/Single/Album because EP is being mis-parsed.
+	//torrentName = fmt.Sprintf("%s [%s] (%s)", title, year, strings.Join(audio, " "))
+	torrentName = fmt.Sprintf("%s [%s] (%s)", title, year, strings.Join(audio, " "))
+
+	rls.ParseString(torrentName)
+	rls.Title = title
+
+	return nil
+}
+
+// IRCParserRedacted parser for Redacted announces
+type IRCParserRedacted struct{}
+
+func (p IRCParserRedacted) Parse(rls *Release, vars map[string]string) error {
+	title := vars["title"]
+	year := vars["year"]
+	releaseTagsString := vars["releaseTags"]
+
+	cleanTags := CleanReleaseTags(releaseTagsString)
+
+	tags := ParseReleaseTagString(cleanTags)
+
+	audio := []string{}
+	if tags.Source != "" {
+		audio = append(audio, tags.Source)
+	}
+	if tags.AudioFormat != "" {
+		audio = append(audio, tags.AudioFormat)
+	}
+	if tags.AudioBitrate != "" {
+		audio = append(audio, tags.AudioBitrate)
+	}
+	rls.Bitrate = tags.AudioBitrate
+	rls.AudioFormat = tags.AudioFormat
+
+	// set log score
+	rls.HasLog = tags.HasLog
+	rls.LogScore = tags.LogScore
+	rls.HasCue = tags.HasCue
+
+	// Construct new release name so we have full control. We remove category such as EP/Single/Album because EP is being mis-parsed.
+	name := fmt.Sprintf("%s [%s] (%s)", title, year, strings.Join(audio, " "))
+
+	rls.ParseString(name)
+	rls.Title = title
 
 	return nil
 }

internal/domain/release.go

@@ -12,7 +12,6 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"os"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -75,6 +74,8 @@ type Release struct {
 	HDR                         []string              `json:"hdr"`
 	Audio                       []string              `json:"-"`
 	AudioChannels               string                `json:"-"`
+	AudioFormat                 string                `json:"-"`
+	Bitrate                     string                `json:"-"`
 	Group                       string                `json:"group"`
 	Region                      string                `json:"-"`
 	Language                    []string              `json:"-"`
@@ -84,6 +85,8 @@ type Release struct {
 	Artists                     string                `json:"-"`
 	Type                        string                `json:"type"` // Album,Single,EP
 	LogScore                    int                   `json:"-"`
+	HasCue                      bool                  `json:"-"`
+	HasLog                      bool                  `json:"-"`
 	Origin                      string                `json:"origin"` // P2P, Internal
 	Tags                        []string              `json:"-"`
 	ReleaseTags                 string                `json:"-"`
@@ -289,6 +292,8 @@ func NewRelease(indexer string) *Release {
 func (r *Release) ParseString(title string) {
 	rel := rls.ParseString(title)
 
+	r.Type = rel.Type.String()
+
 	r.TorrentName = title
 	r.Source = rel.Source
 	r.Resolution = rel.Resolution
@@ -327,18 +332,40 @@ func (r *Release) ParseString(title string) {
 var ErrUnrecoverableError = errors.New("unrecoverable error")
 
 func (r *Release) ParseReleaseTagsString(tags string) {
-	// trim delimiters and closest space
-	re := regexp.MustCompile(`\| |/ |, `)
-	cleanTags := re.ReplaceAllString(tags, "")
-
+	cleanTags := CleanReleaseTags(tags)
 	t := ParseReleaseTagString(cleanTags)
 
 	if len(t.Audio) > 0 {
-		r.Audio = getUniqueTags(r.Audio, t.Audio)
+		//r.Audio = getUniqueTags(r.Audio, t.Audio)
+		r.Audio = t.Audio
+	}
+
+	if t.AudioBitrate != "" {
+		r.Bitrate = t.AudioBitrate
+	}
+
+	if t.AudioFormat != "" {
+		r.AudioFormat = t.AudioFormat
+	}
+
+	if r.AudioChannels == "" && t.Channels != "" {
+		r.AudioChannels = t.Channels
+	}
+
+	if t.HasLog {
+		r.HasLog = true
+
+		if t.LogScore > 0 {
+			r.LogScore = t.LogScore
+		}
+	}
+
+	if t.HasCue {
+		r.HasCue = true
 	}
 
 	if len(t.Bonus) > 0 {
-		if sliceContainsSlice([]string{"Freeleech"}, t.Bonus) {
+		if sliceContainsSlice([]string{"Freeleech", "Freeleech!"}, t.Bonus) {
 			r.Freeleech = true
 		}
 		// TODO handle percent and other types
@@ -363,9 +390,6 @@ func (r *Release) ParseReleaseTagsString(tags string) {
 	if r.Source == "" && t.Source != "" {
 		r.Source = t.Source
 	}
-	if r.AudioChannels == "" && t.Channels != "" {
-		r.AudioChannels = t.Channels
-	}
 }
 
 // ParseSizeBytesString If there are parsing errors, then it keeps the original (or default size 0)