Move to using a lookup for the path resolution

This should resolve issues where paths get "broken" and it's messy in the db
Create migration for db to move to this new structure

add.go

@@ -7,11 +7,9 @@ import (
 	"fmt"
 	"github.com/cfwidget/cfwidget/curseforge"
 	"github.com/cfwidget/cfwidget/env"
-	"github.com/cfwidget/cfwidget/widget"
 	"github.com/spf13/cast"
 	"go.elastic.co/apm/v2"
 	"log"
-	"net/http"
 	"regexp"
 	"strings"
 )
@@ -21,7 +19,7 @@ var FullPathWithId = regexp.MustCompile("[a-zA-Z\\-]+/[a-zA-Z\\-]+/([0-9]+)")
 
 type AddProjectConsumer struct{}
 
-func (consumer *AddProjectConsumer) Consume(url string, ctx context.Context) *widget.Project {
+func (consumer *AddProjectConsumer) Consume(url string, ctx context.Context) *uint {
 	defer func() {
 		err := recover()
 		if err != nil {
@@ -34,68 +32,29 @@ func (consumer *AddProjectConsumer) Consume(url string, ctx context.Context) *wi
 		log.Printf("Resolving path %s", url)
 	}
 
-	var curseId uint
-
-	db, err := GetDatabase()
-	if err != nil {
-		panic(err)
-	}
-
-	db = db.WithContext(ctx)
-
-	project := &widget.Project{}
-	err = db.Where("path = ?", url).Find(&project).Error
-	if err != nil {
-		panic(err)
-	}
-
 	//if the path is just an id, that's the curse id
 	//otherwise..... we can try a search....?
-	if curseId, err = cast.ToUintE(url); err == nil {
-		project.CurseId = &curseId
+	if curseId, err := cast.ToUintE(url); err == nil {
+		return &curseId
 	} else if matches := FullPathWithId.FindStringSubmatch(url); len(matches) > 0 {
 		curseId = cast.ToUint(matches[1])
-		project.CurseId = &curseId
+		return &curseId
 	} else {
 		//for now, we can't resolve, so mark as 404
-		id, err := resolveSlug(url)
-		if id == 0 {
-			project.Status = http.StatusNotFound
-			err = db.Save(project).Error
-			if err != nil {
-				panic(err)
-			}
-			return project
-		} else {
-			project.CurseId = &id
-		}
-	}
-
-	if project.CurseId != nil && *project.CurseId != 0 {
-		var count int64
-		err = db.Model(&widget.Project{}).Where("curse_id = ? AND status = ?", project.CurseId, http.StatusOK).Count(&count).Error
+		id, err := resolveSlug(url, ctx)
 		if err != nil {
 			panic(err)
 		}
-		if count > 0 {
-			project.Status = http.StatusMovedPermanently
+		if id != 0 {
+			return &id
 		}
 	}
 
-	err = db.Save(project).Error
-	if err != nil {
-		panic(err)
-	}
-
-	if project.Status == http.StatusMovedPermanently {
-		return project
-	}
-
-	return SyncProject(project.ID, ctx)
+	return nil
 }
 
-func resolveSlug(path string) (uint, error) {
-	span, ctx := apm.StartSpan(context.Background(), "resolveSlug", "custom")
+func resolveSlug(path string, c context.Context) (uint, error) {
+	span, ctx := apm.StartSpan(c, "resolveSlug", "custom")
 	defer span.End()
 
 	var err error

database.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/cfwidget/cfwidget/env"
 	"github.com/cfwidget/cfwidget/widget"
+	"github.com/go-gormigrate/gormigrate/v2"
 	mysql "go.elastic.co/apm/module/apmgormv2/v2/driver/mysql"
 	"gorm.io/gorm"
 	"log"
@@ -44,11 +45,68 @@ func GetDatabase() (*gorm.DB, error) {
 			db = db.Debug()
 		}
 
-		err = db.AutoMigrate(&widget.Project{}, &widget.Author{})
+		log.Printf("Starting migrations")
+		migrator := gormigrate.New(db, gormigrate.DefaultOptions, []*gormigrate.Migration{
+			{
+				ID: "1682972228",
+				Migrate: func(g *gorm.DB) (err error) {
+					//move old tables away, because they are now considered dead
+					err = g.Migrator().RenameTable("projects", "old_projects")
+					if err != nil {
+						return
+					}
+					err = g.Migrator().RenameTable("authors", "old_authors")
+					if err != nil {
+						return
+					}
+
+					err = g.AutoMigrate(&widget.Project{}, &widget.Author{}, &widget.ProjectLookup{})
+					if err != nil {
+						return
+					}
+
+					//insert our missing data
+					err = g.Exec("INSERT INTO authors (member_id, username, properties, created_at, updated_at) SELECT member_id, username, properties, created_at, updated_at FROM old_authors").Error
+					if err != nil {
+						return
+					}
+
+					err = g.Exec("INSERT INTO projects (id) SELECT DISTINCT curse_id FROM old_projects WHERE properties IS NOT NULL AND STATUS IN (200, 403) AND curse_id IS NOT NULL").Error
+					if err != nil {
+						return
+					}
+
+					err = g.Exec("INSERT INTO project_lookups (path, curse_id) SELECT DISTINCT path, curse_id FROM old_projects").Error
+					if err != nil {
+						return
+					}
+
+					err = g.Exec("UPDATE projects p SET properties = (SELECT properties FROM old_projects op WHERE op.curse_id = p.id AND op.properties IS NOT NULL AND op.STATUS IN (200, 403) ORDER BY id LIMIT 1), STATUS = (SELECT STATUS FROM old_projects op WHERE op.curse_id = p.id AND op.properties IS NOT NULL AND op.STATUS IN (200, 403) ORDER BY id LIMIT 1)").Error
+					if err != nil {
+						return
+					}
+
+					return
+				},
+				Rollback: func(g *gorm.DB) error {
+					//roll back table names, as that is okay
+					_ = g.Migrator().DropTable("projects")
+					_ = g.Migrator().DropTable("authors")
+					_ = g.Migrator().DropTable("project_lookups")
+					_ = g.Migrator().RenameTable("old_projects", "projects")
+					_ = g.Migrator().RenameTable("authors", "authors")
+					return nil
+				},
+			},
+		})
+
+		err = migrator.Migrate()
 		if err != nil {
 			log.Printf("Error connecting to database: %s", err.Error())
 			return nil, err
 		}
+		log.Printf("Migrations complete")
+
 		_db = db
 	}
 

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/gin-contrib/cors v1.4.0
 	github.com/gin-gonic/gin v1.9.0
+	github.com/go-gormigrate/gormigrate/v2 v2.0.2
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0
 	github.com/spf13/cast v1.5.0
 	go.elastic.co/apm/module/apmgin/v2 v2.3.0
@@ -30,7 +31,6 @@ require (
 	github.com/go-playground/validator/v10 v10.12.0 // indirect
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/jcchavezs/porto v0.4.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
@@ -44,7 +44,6 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.7 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
-	github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	go.elastic.co/apm/module/apmsql/v2 v2.3.0 // indirect
@@ -56,8 +55,6 @@ require (
 	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/tools v0.7.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/mysql v1.4.7 // indirect
 	howett.net/plist v1.0.0 // indirect