add tests

.github/dependabot.yaml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "06:00"
+      timezone: "Europe/Amsterdam"

.github/workflows/test.yaml

@@ -0,0 +1,29 @@
+name: Tests
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+    branches:
+      - main
+
+jobs:
+  linter:
+    name: Linter check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24'
+      - name: Run Linter
+        uses: golangci/golangci-lint-action@v8
+  test:
+    name: Regression tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.24'
+      - name: Run tests
+        run: go test ./tests/...

pkg/api.go

@@ -1,53 +1,62 @@
 package pkg
 
 import (
-	"encoding/xml"
+	"log"
 	"net/http"
 	"strings"
 )
 
 type Api struct{}
 
+// healthcheck endpoint, does nothing (useful)
 func (api Api) Ping() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		w.Write([]byte("roxy is running..."))
+		w.Write([]byte("roxy is running...")) //nolint:errcheck
 	}
 }
 
-// /add endpoint for adding rss feeds through api
+func (api Api) Stats(idx *Index) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		data, header := marshall(idx.Urls, JSON)
+		w.Header().Set("Content-Type", header)
+		w.Write(data) //nolint:errcheck
+	}
+}
+
+// endpoint for adding rss feeds through api
 func (api Api) Add(idx *Index) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		category := getStrParam(r.URL, "category")
 		urls := getListParam(r.URL, "urls")
-		tags := getListParam(r.URL, "tags")
 		if len(urls) == 0 {
 			http.Error(w, "no url", http.StatusBadRequest)
 			return
 		}
 		for _, url := range urls {
-			err := idx.Add(url, tags)
+			err := idx.Add(url, category)
 			if err != nil {
 				http.Error(w, "error: "+url, http.StatusInternalServerError)
 				idx.Clear()
 				return
 			}
 		}
-		w.Write([]byte("added " + strings.Join(urls, ", ")))
+		w.Write([]byte("add " + strings.Join(urls, ","))) //nolint:errcheck
 	}
 }
 
-// /get endpoint to query the rss feeds, returns xml in the body
-func (api Api) Get(idx *Index) http.HandlerFunc {
+// query the rss feeds using url parameters, returns xml in the body
+func (api Api) Get(idx *Index, format Format) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		query := Query{
-			Urls:     getListParam(r.URL, "urls"),
-			Tags:     getListParam(r.URL, "tags"),
-			Keywords: getListParam(r.URL, "keywords"),
-			Amount:   getIntParam(r.URL, "amount", 10),
+			Urls:       getListParam(r.URL, "urls"),
+			Categories: getListParam(r.URL, "category"),
+			Keywords:   getListParam(r.URL, "keywords"),
+			Amount:     getIntParam(r.URL, "amount", 10),
 		}
 		result := idx.Get(query)
-		xmlData, _ := xml.MarshalIndent(result, "", "\t")
-		w.Header().Set("Content-Type", "application/xml")
-		w.Write(xmlData)
+		data, header := marshall(result, format)
+		w.Header().Set("Content-Type", header)
+		w.Write(data) //nolint:errcheck
 	}
 }
 
@@ -56,9 +65,10 @@ func (api Api) Refresh(idx *Index) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		idx.Clear()
 		for _, url := range idx.Urls {
-			err := idx.Add(url, []string{}) // TODO! RESET TAGS!
+			log.Printf("refreshing: %s", url.Url)
+			err := idx.Add(url.Url, url.Category)
 			if err != nil {
-				http.Error(w, "can't refresh: "+url, http.StatusInternalServerError)
+				http.Error(w, "fail: "+url.Url, http.StatusInternalServerError)
 				return
 			}
 		}

pkg/index.go

@@ -3,31 +3,32 @@ package pkg
 import (
 	"bufio"
 	"encoding/xml"
-	"fmt"
 	"log"
 	"net/http"
 	"os"
 )
 
 // gets rss feed from a url and adds it to the index, and parses the pubdates
-// TODO: No error return, just skip record and log
-func (idx *Index) Add(url string, tags []string) error {
+func (idx *Index) Add(url string, category string) error {
 	var feed Feed
 	resp, err := http.Get(url) //nolint:errcheck
 	if err != nil {
 		return err
 	}
-	err = xml.NewDecoder(resp.Body).Decode(&feed)
-	if err == nil {
-		feed.ParseTime()
-		feed.Tags = tags
+	if err = xml.NewDecoder(resp.Body).Decode(&feed); err == nil {
+		feed.Category = category
 		feed.Url = url
+		feed.ParseTime()
 		for _, item := range feed.Channel.Items {
 			item.parentFeed = &feed
 			idx.Rank = insertSorted(idx.Rank, &item)
 		}
-		idx.Urls = append(idx.Urls, url)
-		log.Printf("added to feed: '%s'", url)
+		idx.Urls = append(idx.Urls, struct {
+			Url      string
+			Category string
+			Size     int
+		}{url, category, len(feed.Channel.Items)})
+		log.Printf("added to feed: '%s' %v", url, category)
 	}
 	return err
 }
@@ -52,21 +53,23 @@ func (idx *Index) Get(query Query) Result {
 	}
 }
 
-// loags (newsboat) file rss feeds into the index
+// loads (newsboat) file rss feeds into the index
 func (idx *Index) Load(filename string) error {
 	if filename == "" {
-		return nil // nothing to open
+		return nil
 	}
 	file, err := os.Open(filename)
 	if err != nil {
-		return fmt.Errorf("can't open: %s", filename)
+		log.Printf("can't open: %s", filename)
+		return err
 	}
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		line := scanner.Text()
-		url, tags := parseLine(line)
-		if len(url) > 0 {
-			idx.Add(url, tags)
+		if url, category := parseLine(line); len(url) > 0 {
+			if err := idx.Add(url, category); err != nil {
+				log.Printf("error adding url: '%s'", url)
+			}
 		}
 	}
 	return nil
@@ -75,10 +78,12 @@ func (idx *Index) Load(filename string) error {
 // servers all api endpoints for an index instance
 func (idx *Index) Serve(port string) {
 	api := Api{}
-	log.Printf("serving on http://localhost%s", port)
+	log.Printf("serving on: http://localhost%s", port)
 	http.HandleFunc("/", api.Ping())
+	http.HandleFunc("/stats", api.Stats(idx))
 	http.HandleFunc("/add", api.Add(idx))
-	http.HandleFunc("/get", api.Get(idx))
+	http.HandleFunc("/xml", api.Get(idx, XML))
+	http.HandleFunc("/json", api.Get(idx, JSON))
 	http.HandleFunc("/refresh", api.Refresh(idx))
 	log.Fatal(http.ListenAndServe(port, nil))
 }
@@ -90,5 +95,6 @@ func (idx *Index) Clear() {
 
 // initiate rss feed index class (enforce singleton?)
 func NewIndex() *Index {
+	log.Println("starting roxy...")
 	return &Index{}
 }

pkg/rss.go

@@ -2,14 +2,13 @@ package pkg
 
 import (
 	"regexp"
-	"slices"
 	"strings"
 )
 
 // extracts keywords from title, used for querying based on keywords
 func (item *Item) Keywords() []string {
 	re := regexp.MustCompile(`[a-zA-Z]+`)
-	words := re.FindAllString(strings.ToLower(item.Title), -1)
+	words := re.FindAllString(strings.ToLower(item.Description), -1)
 	keywords := []string{}
 	for _, w := range words {
 		if len(w) >= 4 {
@@ -23,11 +22,11 @@ func (item *Item) Keywords() []string {
 // ps, King Terry said case/switch are devine, hence the choice
 func (item *Item) QueryMatch(query Query) bool {
 	switch {
-	case len(query.Urls) > 0 && !slices.Contains(query.Urls, item.parentFeed.Url):
+	case !contains(query.Urls, item.parentFeed.Url):
 		return false
-	case len(query.Tags) > 0 && !overlap(query.Tags, item.parentFeed.Tags):
+	case !contains(query.Categories, item.parentFeed.Category):
 		return false
-	case len(query.Keywords) > 0 && !overlap(query.Keywords, item.Keywords()):
+	case !overlap(query.Keywords, item.Keywords()):
 		return false
 	default:
 		return true

pkg/types.go

@@ -5,24 +5,31 @@ import (
 	"time"
 )
 
+type Format string
+
+const (
+	JSON Format = "json"
+	XML  Format = "xml"
+)
+
 type Item struct {
-	Title       string `xml:"title"`
-	Description string `xml:"description"`
-	Link        string `xml:"link"`
-	Guid        string `xml:"guid"`
-	PubDate     string `xml:"pubDate"`
+	Title       string `xml:"title" json:"title"`
+	Description string `xml:"description" json:"description"`
+	Link        string `xml:"link" json:"link"`
+	Guid        string `xml:"guid" json:"guid"`
+	PubDate     string `xml:"pubDate" json:"pubDate"`
 	// generated
 	timestamp  time.Time
 	parentFeed *Feed
 }
 
 type Channel struct {
-	Title       string   `xml:"title"`
-	Description string   `xml:"description"`
-	Link        string   `xml:"link"`
+	Title       string   `xml:"title" json:"title"`
+	Description string   `xml:"description" json:"description"`
+	Link        string   `xml:"link" json:"link"`
 	Items       []Item   `xml:"item"`
-	PubDate     string   `xml:"pubDate"`
-	Category    []string `xml:"category"`
+	PubDate     string   `xml:"pubDate" json:"pubDate"`
+	Category    []string `xml:"category" json:"category"`
 	Generator   string   `xml:"generator"`
 	// generated
 	timestamp time.Time
@@ -33,28 +40,28 @@ type Feed struct {
 	Version string   `xml:"version,attr"`
 	Channel Channel  `xml:"channel"`
 	// generated
-	Tags []string
-	Url  string
+	Category string
+	Url      string
 }
 
 type Index struct {
 	Rank []*Item
-	Urls []string
-	// Urls []struct {
-	// 	Url  string
-	// 	Tags []string
-	// }
+	Urls []struct {
+		Url      string
+		Category string
+		Size     int
+	}
 }
 
 type Query struct {
-	Urls     []string
-	Tags     []string
-	Keywords []string
-	Amount   int
+	Urls       []string
+	Keywords   []string
+	Categories []string
+	Amount     int
 }
 
 type Result struct {
-	XMLName xml.Name `xml:"rss"`
-	Version string   `xml:"version,attr"`
-	Items   []*Item  `xml:"channel>item"`
+	XMLName xml.Name `xml:"rss" json:"-"`
+	Version string   `xml:"version,attr" json:"-"`
+	Items   []*Item  `xml:"channel>item" json:"items"`
 }