Dev

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17-alpine AS builder
+FROM golang:1.19-alpine AS builder
 
 WORKDIR /cfwidget
 COPY . .
@@ -10,10 +10,6 @@ FROM alpine
 WORKDIR /cfwidget
 
 COPY --from=builder /go/bin/cfwidget /go/bin/cfwidget
-COPY --from=builder /cfwidget/favicon.ico /cfwidget/favicon.ico
-COPY --from=builder /cfwidget/css /cfwidget/css
-COPY --from=builder /cfwidget/js /cfwidget/js
-COPY --from=builder /cfwidget/templates /cfwidget/templates
 
 EXPOSE 8080
 
@@ -22,10 +18,10 @@ ENV DB_HOST="" \
     DB_PASS="" \
     DB_DATABASE="" \
     DB_DEBUG="false" \
-    CACHE_TTL="5m" \
+    CACHE_TTL="1h" \
     CORE_KEY_FILE="/run/secrets/core_key" \
     CORE_KEY="" \
-    WEB_HOSTNAME="www.localhost" \
+    API_HOSTNAME="api.localhost:8080" \
     DEBUG="false" \
     GIN_MODE="release" \
     ELASTIC_APM_SERVER_URL="" \

add.go

@@ -7,11 +7,9 @@ import (
 	"fmt"
 	"github.com/cfwidget/cfwidget/curseforge"
 	"github.com/cfwidget/cfwidget/env"
-	"github.com/cfwidget/cfwidget/widget"
 	"github.com/spf13/cast"
 	"go.elastic.co/apm/v2"
 	"log"
-	"net/http"
 	"regexp"
 	"strings"
 )
@@ -21,7 +19,7 @@ var FullPathWithId = regexp.MustCompile("[a-zA-Z\\-]+/[a-zA-Z\\-]+/([0-9]+)")
 
 type AddProjectConsumer struct{}
 
-func (consumer *AddProjectConsumer) Consume(url string, ctx context.Context) *widget.Project {
+func (consumer *AddProjectConsumer) Consume(url string, ctx context.Context) *uint {
 	defer func() {
 		err := recover()
 		if err != nil {
@@ -34,68 +32,28 @@ func (consumer *AddProjectConsumer) Consume(url string, ctx context.Context) *wi
 		log.Printf("Resolving path %s", url)
 	}
 
-	var curseId uint
-
-	db, err := GetDatabase()
-	if err != nil {
-		panic(err)
-	}
-
-	db = db.WithContext(ctx)
-
-	project := &widget.Project{}
-	err = db.Where("path = ?", url).Find(&project).Error
-	if err != nil {
-		panic(err)
-	}
-
 	//if the path is just an id, that's the curse id
 	//otherwise..... we can try a search....?
-	if curseId, err = cast.ToUintE(url); err == nil {
-		project.CurseId = &curseId
+	if curseId, err := cast.ToUintE(url); err == nil {
+		return &curseId
 	} else if matches := FullPathWithId.FindStringSubmatch(url); len(matches) > 0 {
 		curseId = cast.ToUint(matches[1])
-		project.CurseId = &curseId
+		return &curseId
 	} else {
-		//for now, we can't resolve, so mark as 404
-		id, err := resolveSlug(url)
-		if id == 0 {
-			project.Status = http.StatusNotFound
-			err = db.Save(project).Error
-			if err != nil {
-				panic(err)
-			}
-			return project
-		} else {
-			project.CurseId = &id
-		}
-	}
-
-	if project.CurseId != nil && *project.CurseId != 0 {
-		var count int64
-		err = db.Model(&widget.Project{}).Where("curse_id = ? AND status IN (?, ?)", project.CurseId, http.StatusOK, http.StatusForbidden).Count(&count).Error
+		id, err := resolveSlug(url, ctx)
 		if err != nil {
 			panic(err)
 		}
-		if count > 0 {
-			project.Status = http.StatusMovedPermanently
+		if id != 0 {
+			return &id
 		}
 	}
 
-	err = db.Save(project).Error
-	if err != nil {
-		panic(err)
-	}
-
-	if project.Status == http.StatusMovedPermanently {
-		return project
-	}
-
-	return SyncProject(project.ID, ctx)
+	return nil
 }
 
-func resolveSlug(path string) (uint, error) {
-	span, ctx := apm.StartSpan(context.Background(), "resolveSlug", "custom")
+func resolveSlug(path string, c context.Context) (uint, error) {
+	span, ctx := apm.StartSpan(c, "resolveSlug", "custom")
 	defer span.End()
 
 	var err error

css/app.css

@@ -1306,7 +1306,7 @@ img {
 }
 
 .mw7 {
-    max-width: 48rem
+    max-width: 60rem
 }
 
 .mw8 {
@@ -3183,7 +3183,7 @@ img {
 }
 
 .measure-wide {
-    max-width: 34em
+    max-width: 60em
 }
 
 .measure-narrow {

curseforge/client.go

@@ -9,11 +9,10 @@ import (
 	"github.com/cfwidget/cfwidget/env"
 	"go.elastic.co/apm/module/apmhttp/v2"
 	"go.elastic.co/apm/v2"
+	"image"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
-	"net/url"
 	"os"
 	"time"
 )
@@ -52,26 +51,21 @@ func StartGameCacheSyncer() {
 func Call(u string, ctx context.Context) (*http.Response, error) {
 	key := os.Getenv("CORE_KEY")
 
-	path, err := url.Parse(u)
+	request, err := http.NewRequestWithContext(ctx, "GET", u, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	request := &http.Request{
-		Method: "GET",
-		URL:    path,
-		Header: http.Header{},
-	}
 	request.Header.Add("x-api-key", key)
 
-	response, err := client.Do(request.WithContext(ctx))
+	response, err := client.Do(request)
 
 	if env.GetBool("DEBUG") {
 		//clone body so we can "replace" it
 		body, _ := io.ReadAll(response.Body)
 		_ = response.Body.Close()
-		response.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-		log.Printf("URL %s\nResult: %s\nBody: %s\n", path.String(), response.Status, string(body))
+		response.Body = io.NopCloser(bytes.NewBuffer(body))
+		log.Printf("URL %s\nResult: %s\nBody: %s\n", u, response.Status, string(body))
 	}
 
 	return response, err
@@ -262,3 +256,19 @@ func getFilesForPage(projectId, page uint, ctx context.Context) (FilesResponse,
 	err = json.NewDecoder(response.Body).Decode(&files)
 	return files, err
 }
+
+func GetThumbnail(url string, ctx context.Context) (image.Image, error) {
+	request, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	img, _, err := image.Decode(response.Body)
+	return img, err
+}

database.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/cfwidget/cfwidget/env"
 	"github.com/cfwidget/cfwidget/widget"
+	"github.com/go-gormigrate/gormigrate/v2"
 	mysql "go.elastic.co/apm/module/apmgormv2/v2/driver/mysql"
 	"gorm.io/gorm"
 	"log"
@@ -44,11 +45,73 @@ func GetDatabase() (*gorm.DB, error) {
 			db = db.Debug()
 		}
 
-		err = db.AutoMigrate(&widget.Project{}, &widget.Author{})
+		log.Printf("Starting migrations")
+		migrator := gormigrate.New(db, gormigrate.DefaultOptions, []*gormigrate.Migration{
+			{
+				ID: "1682972228",
+				Migrate: func(g *gorm.DB) (err error) {
+					if !g.Migrator().HasTable("projects") {
+						err = g.AutoMigrate(&widget.Project{}, &widget.Author{}, &widget.ProjectLookup{})
+						return
+					}
+
+					//move old tables away, because they are now considered dead
+					err = g.Migrator().RenameTable("projects", "old_projects")
+					if err != nil {
+						return
+					}
+					err = g.Migrator().RenameTable("authors", "old_authors")
+					if err != nil {
+						return
+					}
+
+					err = g.AutoMigrate(&widget.Project{}, &widget.Author{}, &widget.ProjectLookup{})
+					if err != nil {
+						return
+					}
+
+					//insert our missing data
+					err = g.Exec("INSERT INTO authors (member_id, username, properties, created_at, updated_at) SELECT member_id, username, properties, created_at, updated_at FROM old_authors").Error
+					if err != nil {
+						return
+					}
+
+					err = g.Exec("INSERT INTO projects (id) SELECT DISTINCT curse_id FROM old_projects WHERE properties IS NOT NULL AND STATUS IN (200, 403) AND curse_id IS NOT NULL").Error
+					if err != nil {
+						return
+					}
+
+					err = g.Exec("INSERT INTO project_lookups (path, curse_id) SELECT DISTINCT path, curse_id FROM old_projects").Error
+					if err != nil {
+						return
+					}
+
+					err = g.Exec("UPDATE projects p SET properties = (SELECT properties FROM old_projects op WHERE op.curse_id = p.id AND op.properties IS NOT NULL AND op.STATUS IN (200, 403) ORDER BY id LIMIT 1), STATUS = (SELECT STATUS FROM old_projects op WHERE op.curse_id = p.id AND op.properties IS NOT NULL AND op.STATUS IN (200, 403) ORDER BY id LIMIT 1)").Error
+					if err != nil {
+						return
+					}
+
+					return
+				},
+				Rollback: func(g *gorm.DB) error {
+					//roll back table names, as that is okay
+					_ = g.Migrator().DropTable("projects")
+					_ = g.Migrator().DropTable("authors")
+					_ = g.Migrator().DropTable("project_lookups")
+					_ = g.Migrator().RenameTable("old_projects", "projects")
+					_ = g.Migrator().RenameTable("authors", "authors")
+					return nil
+				},
+			},
+		})
+
+		err = migrator.Migrate()
 		if err != nil {
 			log.Printf("Error connecting to database: %s", err.Error())
 			return nil, err
 		}
+		log.Printf("Migrations complete")
+
 		_db = db
 	}
 

docker-compose.yml

@@ -15,6 +15,7 @@ services:
       DEBUG: "false"
       CORE_KEY_FILE: "/run/secrets/core_key"
       WEB_HOSTNAME: "${WEB_HOST}"
+      API_HOSTNAME: "${API_HOST}"
       CACHE_TTL: "1h"
     secrets:
       - core_key
@@ -25,13 +26,13 @@ services:
     deploy:
       labels:
         - "traefik.enable=true"
-        - "traefik.http.routers.${SERVICE_NAME}.rule=Host(`${WEB_HOST}`) || Host(`${API_HOST}`) || Host(`${ROOT_HOST}`)"
+        - "traefik.http.routers.${SERVICE_NAME}.rule=Host(`${WEB_HOST}`) || Host(`${API_HOST}`) || Host(`www.${WEB_HOST}`)"
         - "traefik.http.routers.${SERVICE_NAME}.entrypoints=websecure"
         - "traefik.http.routers.${SERVICE_NAME}.tls.certresolver=myresolver"
         - "traefik.http.services.${SERVICE_NAME}.loadbalancer.server.port=8080"
 
   database:
-    image: mariadb
+    image: mariadb:10.11
     restart: always
     environment:
       MYSQL_USER: widget