Scanner: Configuration File + Parse Metadata from paths (#44)

.github/workflows/scanner.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Setup Go
         uses: ./.github/actions/cache-scanner
       - name: Install Dependencies and Build
-        run: go build
+        run: go build -o scanner ./app
   Format:
     if: ${{ needs.changes.outputs.scanner == 'true' && github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
@@ -53,6 +53,28 @@ jobs:
         with:
           gofmt-path: './scanner'
           gofmt-flags: '-l'
+  Tests:
+    runs-on: ubuntu-latest
+    needs: [ Build ]
+    if: ${{ needs.changes.outputs.scanner == 'true' && always() }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - name: Setup Go
+        uses: ./.github/actions/cache-scanner
+      - name: Install Coverage Report
+        run: go install github.com/jandelgado/gcov2lcov@latest
+      - name: Run tests
+        run: go test ./... -coverprofile=coverage.out
+      - name: Convert Coverage file
+        run: gcov2lcov -infile=coverage.out -outfile=coverage.lcov
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./scanner/coverage.lcov
+          flags: scanner
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   Dockerize:
     if: ${{ needs.changes.outputs.scanner == 'true' && github.event_name == 'pull_request' }}
     needs: [ Build ]

.gitignore

@@ -1 +1,2 @@
-.env
+.env
+scanner.json

docker-compose.dev.yml

@@ -6,10 +6,12 @@ services:
     restart: on-failure
     volumes:
       - ./scanner:/app
+      - ./scanner.example.json:/config/scanner.json
       - ${DATA_DIR}:/video
     environment:
       - WATCH_DIR=/video
       - SCANNER_API_KEY=${SCANNER_API_KEY}
+      - CONFIG_DIR=/config
   api:
     build:
       context: ./api

docker-compose.yml

@@ -29,9 +29,11 @@ services:
         condition: service_healthy
     volumes:
       - ${DATA_DIR}:/video
+      - ./scanner.json:/app/scanner.json
     environment:
       - WATCH_DIR=/video
       - SCANNER_API_KEY=${SCANNER_API_KEY}
+      - CONFIG_DIR=/app
   db:
     image: postgres:alpine3.16
     healthcheck:

scanner.example.json

@@ -0,0 +1,13 @@
+{
+	"regexes": {
+		"movies": [
+			"[\\/\\\\]videos[\\/\\\\](?P<PackageArtist>[^\\/\\\\]+)([\\/\\\\]+|(\\s+-\\s+))(?P<Package>[^\\/\\\\]+?)(\\s+\\((?P<PackageYear>\\d+)\\))?[\\/\\\\]+((?P<Artist>[^\\/\\\\]+)\\s+-\\s+)?(?P<Movie>[^\\/\\\\]+?)(\\s+\\((?P<Year>\\d+)\\))?\\..+",
+			"[\\/\\\\]videos[\\/\\\\](?P<PackageArtist>(?P<Artist>[^\\/\\\\]+))(\\s+-\\s+)(?P<Package>(?P<Movie>[^\\/\\\\]+?))(\\s+\\((?P<PackageYear>(?P<Year>\\d+))\\))?\\..+"
+		],
+		"extras": [
+			"[\\/\\\\]videos[\\/\\\\]((?P<PackageArtist>[^\\/\\\\]+)((\\s+-\\s+)|([\\/\\\\]+))(([^\\/\\\\]+)\\s+-\\s+)?((?P<Package>[^\\/\\\\]+?)))(\\s+\\((?P<PackageYear>\\d+)\\))?[\\/\\\\]+(Extra(s)?[\\/\\\\]+)(((?P<Disc>\\d+)-)?(?P<Index>\\d+)\\s+)?(?P<Extra>[^\\/\\\\]+?)(?P<PlexExtraType>-[a-z]+)?\\..+",
+			"[\\/\\\\]videos[\\/\\\\]((?P<PackageArtist>[^\\/\\\\]+)((\\s+-\\s+)|([\\/\\\\]+))(([^\\/\\\\]+)\\s+-\\s+)?((?P<Package>[^\\/\\\\]+?)))(\\s+\\((?P<PackageYear>\\d+)\\))?[\\/\\\\]+(Extra(s)?[\\/\\\\]+)?(((?P<Disc>\\d+)-)?(?P<Index>\\d+)\\s+)?(?P<Extra>[^\\/\\\\]+?)(?P<PlexExtraType>-[a-z]+)\\..+",
+			"[\\/\\\\]videos[\\/\\\\]((?P<PackageArtist>[^\\/\\\\]+)((\\s+-\\s+)|([\\/\\\\]+))(([^\\/\\\\]+)\\s+-\\s+)?((?P<Package>[^\\/\\\\]+?)))(\\s+\\((?P<PackageYear>\\d+)\\))?[\\/\\\\]+(Extra(s)?[\\/\\\\]+)?(((?P<Disc>\\d+)-)?(?P<Index>\\d+)\\s+)(?P<Extra>[^\\/\\\\]+?)(?P<PlexExtraType>-[a-z]+)?\\..+"
+		]
+	}
+}

scanner/.gitignore

@@ -1 +1,2 @@
-scanner
+scanner
+coverage.out

scanner/Dockerfile

@@ -2,10 +2,11 @@ FROM golang:1.22.1-alpine as builder
 WORKDIR /app
 COPY go.mod go.sum ./
 RUN go mod download
-COPY *.go ./
-RUN CGO_ENABLED=0 GOOS=linux go build -o ./scanner
+COPY ./app ./app
+COPY ./pkg ./pkg
+RUN CGO_ENABLED=0 GOOS=linux go build -o ./scanner ./app
 
 FROM debian:bullseye-slim as runner
 WORKDIR /app
 COPY --from=builder /app/scanner ./
-CMD ./scanner -d "${WATCH_DIR}"
+CMD ./scanner -d "${WATCH_DIR}" -c "$CONFIG_DIR"/scanner.json

scanner/Dockerfile.dev

@@ -1,4 +1,4 @@
 FROM golang:1.22.1-alpine
 WORKDIR /app
 RUN go install github.com/githubnemo/CompileDaemon@latest
-ENTRYPOINT CompileDaemon -build="go build -o ./scanner" -command="./scanner -d $WATCH_DIR"
+ENTRYPOINT CompileDaemon -build="go build -o ./scanner ./app" -command="./scanner -d $WATCH_DIR -c $CONFIG_DIR/scanner.json"

scanner/main.go → scanner/app/main.go

@@ -2,13 +2,14 @@ package main
 
 import (
 	"fmt"
+	"github.com/Arthi-chaud/Blee/scanner/pkg/config"
 	"github.com/kpango/glg"
 	"github.com/radovskyb/watcher"
 	"os"
 	"time"
 )
 
-func setup_logger() {
+func setupLogger() {
 	glg.Get().
 		SetMode(glg.STD).
 		// We will be watching the logs through docker-compose
@@ -18,8 +19,8 @@ func setup_logger() {
 }
 
 func main() {
-	setup_logger()
-	c := get_config()
+	setupLogger()
+	c := config.GetConfig()
 	w := watcher.New()
 	go func() {
 		for {

scanner/go.mod

@@ -1,4 +1,4 @@
-module scanner
+module github.com/Arthi-chaud/Blee/scanner
 
 go 1.22.1
 
@@ -8,6 +8,24 @@ require (
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/glenn-brown/golang-pkg-pcre v0.0.0-20120522223659-48bb82a8b8ce // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+require (
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.19.0
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/kpango/fastime v1.1.9 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/stretchr/testify v1.9.0
+	gopkg.in/go-playground/validator.v9 v9.31.0 // indirect
 )

scanner/go.sum

@@ -1,8 +1,39 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/glenn-brown/golang-pkg-pcre v0.0.0-20120522223659-48bb82a8b8ce h1:MS/JOOAHf4U2iKl8+1+vzUcG9t9ru1hnZJ9NEBDvMnY=
+github.com/glenn-brown/golang-pkg-pcre v0.0.0-20120522223659-48bb82a8b8ce/go.mod h1:5385NDJ+Gt5loLrAlc8Rr5lKA1L5BE5O94jfdwEX9kg=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=
+github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/kpango/fastime v1.1.9 h1:xVQHcqyPt5M69DyFH7g1EPRns1YQNap9d5eLhl/Jy84=
 github.com/kpango/fastime v1.1.9/go.mod h1:vyD7FnUn08zxY4b/QFBZVG+9EWMYsNl+QF0uE46urD4=
 github.com/kpango/glg v1.6.15 h1:nw0xSxpSyrDIWHeb3dvnE08PW+SCbK+aYFETT75IeLA=
 github.com/kpango/glg v1.6.15/go.mod h1:cmsc7Yeu8AS3wHLmN7bhwENXOpxfq+QoqxCIk2FneRk=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE=
 github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M=
+gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

scanner/pkg/config/config.go

@@ -0,0 +1,72 @@
+package config
+
+import (
+	"flag"
+	"github.com/goccy/go-json"
+	"github.com/kpango/glg"
+	"gopkg.in/go-playground/validator.v9"
+	"os"
+)
+
+type Config struct {
+	// The Directory to watch
+	WatchDir string
+	// Secret API Key to authenticate
+	ApiKey string
+	// User configuration from `scanner.json`
+	UserConfig UserConfiguration
+}
+
+type UserConfiguration struct {
+	Regexes struct {
+		// Regexes of the extras to match
+		Extra []string `json:"extras" validate:"required"`
+		// Regexes of the movies to match
+		Movie []string `json:"movies" validate:"required"`
+	} `json:"regexes" validate:"required,dive,required"`
+}
+
+func parseConfigFile(file string) UserConfiguration {
+	bytes, err := os.ReadFile(file)
+
+	if err != nil {
+		glg.Fatalf("Could not read configuration file: %s", err)
+		os.Exit(1)
+	}
+
+	var config UserConfiguration
+
+	json.Unmarshal(bytes, &config)
+	validation_error := validator.New().Struct(config)
+	if validation_error != nil {
+		glg.Fatalf("An error occured while validating configuration file: %s", validation_error)
+		os.Exit(1)
+	}
+	return config
+}
+
+// Parses and return a config from the CLI args and env args
+func GetConfig() Config {
+	var config Config
+	watchDir := flag.String("d", "", "the directory to watch")
+	configFilePath := flag.String("c", "", "the path to the `scanner.json`")
+	flag.Parse()
+
+	if len(*watchDir) == 0 || len(*configFilePath) == 0 {
+		glg.Fatalf("Missing argument. Run with `-h` for usage.")
+		os.Exit(1)
+	}
+
+	apiKey, is_present := os.LookupEnv("SCANNER_API_KEY")
+
+	if !is_present || len(apiKey) == 0 {
+		glg.Fatalf("SCANNER_API_KEY is missing or empty.")
+		os.Exit(1)
+	}
+
+	config.ApiKey = apiKey
+	config.WatchDir = *watchDir
+	config.UserConfig = parseConfigFile(*configFilePath)
+	glg.Log("Configuration parsed successfully")
+	return config
+}

scanner/pkg/models/types.go

@@ -0,0 +1,21 @@
+package models
+
+type ExtraType int64
+
+const (
+	AlternateView ExtraType = iota
+	Backdrops
+	BehindTheScenes
+	Interview
+	MusicVideo
+	Other
+	Performance
+	Trailer
+)
+
+type MovieType int64
+
+const (
+	Concert MovieType = iota
+	Documentary
+)