From 4a520a69ebb205874dc743e4a0a76a74111e1314 Mon Sep 17 00:00:00 2001 From: TimoKats Date: Sat, 18 Apr 2026 21:06:53 +0200 Subject: [PATCH 1/8] add tests --- {tests => .docker}/.gitkeep | 0 .github/dependabot.yaml | 9 +++++ .github/workflows/test.yaml | 29 ++++++++++++++ tests/regression_test.go | 79 +++++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+) rename {tests => .docker}/.gitkeep (100%) create mode 100644 .github/dependabot.yaml create mode 100644 .github/workflows/test.yaml create mode 100644 tests/regression_test.go diff --git a/tests/.gitkeep b/.docker/.gitkeep similarity index 100% rename from tests/.gitkeep rename to .docker/.gitkeep diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..7d547b1 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,9 @@ +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "06:00" + timezone: "Europe/Amsterdam" \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..49add6c --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,29 @@ +name: tests + +on: + pull_request: + types: [opened, reopened, synchronize] + branches: + - main + +jobs: + linter: + name: linter check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.24' + - name: golangci-lint + uses: golangci/golangci-lint-action@v8 + test: + name: regression tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.24' + - name: regression tests + run: go test ./test/... \ No newline at end of file diff --git a/tests/regression_test.go b/tests/regression_test.go new file mode 100644 index 0000000..bc851a0 --- /dev/null +++ b/tests/regression_test.go @@ -0,0 +1,79 @@ +package tests + +import ( + "encoding/xml" + "net/http" + "testing" + "time" + + "github.com/TimoKats/roxy/pkg" +) + +func TestAdd(t *testing.T) { + idx := pkg.NewIndex() + url := "https://timokats.xyz/feed/website.xml" + tags := []string{"example", "test"} + err := idx.Add(url, tags) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if len(idx.Rank) == 0 { + t.Error("Rank should not be empty after adding an item") + } +} + +func TestGet(t *testing.T) { + idx := pkg.NewIndex() + url := "https://timokats.xyz/feed/website.xml" + tags := []string{"example", "test"} + err := idx.Add(url, tags) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + query := pkg.Query{Amount: 10} + result := idx.Get(query) + if len(result.Items) == 0 { + t.Error("Result should contain items after querying") + } +} + +func TestServe(t *testing.T) { + idx := pkg.NewIndex() + url := "https://timokats.xyz/feed/website.xml" + tags := []string{"example", "test"} + err := idx.Add(url, tags) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + go idx.Serve(":8080") + time.Sleep(2 * time.Second) + + resp, err := http.Get("http://localhost:8080/get?amount=10") + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + defer resp.Body.Close() + + result := pkg.Result{} + err = xml.NewDecoder(resp.Body).Decode(&result) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if len(result.Items) == 0 { + t.Error("Result should contain items after querying via API") + } +} + +func TestClear(t *testing.T) { + idx := pkg.NewIndex() + url := "https://timokats.xyz/feed/website.xml" + tags := []string{"example", "test"} + err := idx.Add(url, tags) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + idx.Clear() + if len(idx.Rank) > 0 { + t.Error("Rank should be empty after clearing") + } +} From f3cd4c559f0072cca6337971a5904c6bd8afa5e3 Mon Sep 17 00:00:00 2001 From: TimoKats Date: Sat, 18 Apr 2026 21:08:00 +0200 Subject: [PATCH 2/8] fix filename in test --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 49add6c..ede70a4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -26,4 +26,4 @@ jobs: with: go-version: '1.24' - name: regression tests - run: go test ./test/... \ No newline at end of file + run: go test ./tests/... \ No newline at end of file From 024dfc00a1987abb95bd8d735a43ffd0192fecd2 Mon Sep 17 00:00:00 2001 From: TimoKats Date: Sat, 18 Apr 2026 21:10:33 +0200 Subject: [PATCH 3/8] rename workflows --- .github/workflows/test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ede70a4..6bddb53 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,4 +1,4 @@ -name: tests +name: Tests on: pull_request: @@ -8,7 +8,7 @@ on: jobs: linter: - name: linter check + name: Linter check runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -18,7 +18,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v8 test: - name: regression tests + name: Regression tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 6f30f106f7d1071235ca42e8bd9dc18ea19dfb16 Mon Sep 17 00:00:00 2001 From: TimoKats Date: Sat, 18 Apr 2026 21:14:07 +0200 Subject: [PATCH 4/8] some moar renaming, because I'm crazyu --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6bddb53..b161baa 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,7 +15,7 @@ jobs: - uses: actions/setup-go@v5 with: go-version: '1.24' - - name: golangci-lint + - name: Run Linter uses: golangci/golangci-lint-action@v8 test: name: Regression tests @@ -25,5 +25,5 @@ jobs: - uses: actions/setup-go@v4 with: go-version: '1.24' - - name: regression tests + - name: Run tests run: go test ./tests/... \ No newline at end of file From b9a3ce7ae19c932f02ca1fc51b4be16e5ba8b254 Mon Sep 17 00:00:00 2001 From: TimoKats Date: Sat, 18 Apr 2026 22:13:07 +0200 Subject: [PATCH 5/8] change some things/ --- pkg/api.go | 5 ++++- pkg/index.go | 25 ++++++++++++------------- main.go => roxy.go | 7 ++++--- 3 files changed, 20 insertions(+), 17 deletions(-) rename main.go => roxy.go (65%) diff --git a/pkg/api.go b/pkg/api.go index 49856b4..2bb19a6 100644 --- a/pkg/api.go +++ b/pkg/api.go @@ -2,19 +2,21 @@ package pkg import ( "encoding/xml" + "log" "net/http" "strings" ) type Api struct{} +// healthcheck endpoint func (api Api) Ping() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("roxy is running...")) } } -// /add endpoint for adding rss feeds through api +// endpoint for adding rss feeds through api func (api Api) Add(idx *Index) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { urls := getListParam(r.URL, "urls") @@ -56,6 +58,7 @@ func (api Api) Refresh(idx *Index) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { idx.Clear() for _, url := range idx.Urls { + log.Printf("refreshing: %s", url) err := idx.Add(url, []string{}) // TODO! RESET TAGS! if err != nil { http.Error(w, "can't refresh: "+url, http.StatusInternalServerError) diff --git a/pkg/index.go b/pkg/index.go index ca99a54..23d452e 100644 --- a/pkg/index.go +++ b/pkg/index.go @@ -3,22 +3,19 @@ package pkg import ( "bufio" "encoding/xml" - "fmt" "log" "net/http" "os" ) // gets rss feed from a url and adds it to the index, and parses the pubdates -// TODO: No error return, just skip record and log func (idx *Index) Add(url string, tags []string) error { var feed Feed resp, err := http.Get(url) //nolint:errcheck if err != nil { return err } - err = xml.NewDecoder(resp.Body).Decode(&feed) - if err == nil { + if err = xml.NewDecoder(resp.Body).Decode(&feed); err == nil { feed.ParseTime() feed.Tags = tags feed.Url = url @@ -27,7 +24,7 @@ func (idx *Index) Add(url string, tags []string) error { idx.Rank = insertSorted(idx.Rank, &item) } idx.Urls = append(idx.Urls, url) - log.Printf("added to feed: '%s'", url) + log.Printf("added to feed: '%s' %v", url, tags) } return err } @@ -53,29 +50,30 @@ func (idx *Index) Get(query Query) Result { } // loags (newsboat) file rss feeds into the index -func (idx *Index) Load(filename string) error { +func (idx *Index) Load(filename string) { if filename == "" { - return nil // nothing to open + return } file, err := os.Open(filename) if err != nil { - return fmt.Errorf("can't open: %s", filename) + log.Printf("can't open: %s", filename) + return } scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() - url, tags := parseLine(line) - if len(url) > 0 { - idx.Add(url, tags) + if url, tags := parseLine(line); len(url) > 0 { + if err := idx.Add(url, tags); err != nil { + log.Printf("error adding url: '%s'", url) + } } } - return nil } // servers all api endpoints for an index instance func (idx *Index) Serve(port string) { api := Api{} - log.Printf("serving on http://localhost%s", port) + log.Printf("serving on: http://localhost%s", port) http.HandleFunc("/", api.Ping()) http.HandleFunc("/add", api.Add(idx)) http.HandleFunc("/get", api.Get(idx)) @@ -90,5 +88,6 @@ func (idx *Index) Clear() { // initiate rss feed index class (enforce singleton?) func NewIndex() *Index { + log.Println("starting roxy...") return &Index{} } diff --git a/main.go b/roxy.go similarity index 65% rename from main.go rename to roxy.go index cffb8ce..e8b13c2 100644 --- a/main.go +++ b/roxy.go @@ -2,15 +2,16 @@ package main import ( "flag" - "log" pkg "github.com/TimoKats/roxy/pkg" ) func main() { - filename := flag.String("filename", "", "(newsboat) file with rss feeds") + // flags + filename := flag.String("feeds", "", "(newsboat) file with rss feeds") port := flag.String("port", "2112", "port number to serve on") - log.Println("starting roxy...") + flag.Parse() + // start server idx := pkg.NewIndex() idx.Load(*filename) idx.Serve(":" + *port) From 9d8961b135088c102727b2ae2ddaf15b0c95aabe Mon Sep 17 00:00:00 2001 From: TimoKats Date: Sat, 18 Apr 2026 22:19:52 +0200 Subject: [PATCH 6/8] no linter errors when not needed --- pkg/api.go | 6 +++--- pkg/index.go | 2 +- tests/regression_test.go | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/api.go b/pkg/api.go index 2bb19a6..69934c1 100644 --- a/pkg/api.go +++ b/pkg/api.go @@ -12,7 +12,7 @@ type Api struct{} // healthcheck endpoint func (api Api) Ping() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("roxy is running...")) + w.Write([]byte("roxy is running...")) //nolint:errcheck } } @@ -33,7 +33,7 @@ func (api Api) Add(idx *Index) http.HandlerFunc { return } } - w.Write([]byte("added " + strings.Join(urls, ", "))) + w.Write([]byte("added " + strings.Join(urls, ", "))) //nolint:errcheck } } @@ -49,7 +49,7 @@ func (api Api) Get(idx *Index) http.HandlerFunc { result := idx.Get(query) xmlData, _ := xml.MarshalIndent(result, "", "\t") w.Header().Set("Content-Type", "application/xml") - w.Write(xmlData) + w.Write(xmlData) //nolint:errcheck } } diff --git a/pkg/index.go b/pkg/index.go index 23d452e..7e5f578 100644 --- a/pkg/index.go +++ b/pkg/index.go @@ -49,7 +49,7 @@ func (idx *Index) Get(query Query) Result { } } -// loags (newsboat) file rss feeds into the index +// loads (newsboat) file rss feeds into the index func (idx *Index) Load(filename string) { if filename == "" { return diff --git a/tests/regression_test.go b/tests/regression_test.go index bc851a0..5bb4411 100644 --- a/tests/regression_test.go +++ b/tests/regression_test.go @@ -45,14 +45,13 @@ func TestServe(t *testing.T) { if err != nil { t.Errorf("Expected no error, got %v", err) } + go idx.Serve(":8080") time.Sleep(2 * time.Second) - resp, err := http.Get("http://localhost:8080/get?amount=10") if err != nil { t.Errorf("Expected no error, got %v", err) } - defer resp.Body.Close() result := pkg.Result{} err = xml.NewDecoder(resp.Body).Decode(&result) From 8b363d2f4b8dc6118638740c6c463cf539f9099d Mon Sep 17 00:00:00 2001 From: TimoKats Date: Thu, 23 Apr 2026 21:04:31 +0200 Subject: [PATCH 7/8] use category --- pkg/api.go | 16 ++++++++-------- pkg/index.go | 21 +++++++++++---------- pkg/rss.go | 4 ++-- pkg/types.go | 15 +++++++-------- pkg/utils.go | 22 ++++++++++++++++------ roxy.go | 6 +++++- tests/regression_test.go | 16 ++++------------ 7 files changed, 53 insertions(+), 47 deletions(-) diff --git a/pkg/api.go b/pkg/api.go index 69934c1..5085713 100644 --- a/pkg/api.go +++ b/pkg/api.go @@ -9,7 +9,7 @@ import ( type Api struct{} -// healthcheck endpoint +// healthcheck endpoint, does nothing (useful) func (api Api) Ping() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("roxy is running...")) //nolint:errcheck @@ -19,30 +19,30 @@ func (api Api) Ping() http.HandlerFunc { // endpoint for adding rss feeds through api func (api Api) Add(idx *Index) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + category := getStrParam(r.URL, "category") urls := getListParam(r.URL, "urls") - tags := getListParam(r.URL, "tags") if len(urls) == 0 { http.Error(w, "no url", http.StatusBadRequest) return } for _, url := range urls { - err := idx.Add(url, tags) + err := idx.Add(url, category) if err != nil { http.Error(w, "error: "+url, http.StatusInternalServerError) idx.Clear() return } } - w.Write([]byte("added " + strings.Join(urls, ", "))) //nolint:errcheck + w.Write([]byte("add " + strings.Join(urls, ","))) //nolint:errcheck } } -// /get endpoint to query the rss feeds, returns xml in the body +// query the rss feeds using url parameters, returns xml in the body func (api Api) Get(idx *Index) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { query := Query{ Urls: getListParam(r.URL, "urls"), - Tags: getListParam(r.URL, "tags"), + Category: getStrParam(r.URL, "category"), Keywords: getListParam(r.URL, "keywords"), Amount: getIntParam(r.URL, "amount", 10), } @@ -59,9 +59,9 @@ func (api Api) Refresh(idx *Index) http.HandlerFunc { idx.Clear() for _, url := range idx.Urls { log.Printf("refreshing: %s", url) - err := idx.Add(url, []string{}) // TODO! RESET TAGS! + err := idx.Add(url.Url, url.Category) if err != nil { - http.Error(w, "can't refresh: "+url, http.StatusInternalServerError) + http.Error(w, "fail: "+url.Url, http.StatusInternalServerError) return } } diff --git a/pkg/index.go b/pkg/index.go index 7e5f578..12db0be 100644 --- a/pkg/index.go +++ b/pkg/index.go @@ -9,22 +9,22 @@ import ( ) // gets rss feed from a url and adds it to the index, and parses the pubdates -func (idx *Index) Add(url string, tags []string) error { +func (idx *Index) Add(url string, category string) error { var feed Feed resp, err := http.Get(url) //nolint:errcheck if err != nil { return err } if err = xml.NewDecoder(resp.Body).Decode(&feed); err == nil { - feed.ParseTime() - feed.Tags = tags + feed.Category = category feed.Url = url + feed.ParseTime() for _, item := range feed.Channel.Items { item.parentFeed = &feed idx.Rank = insertSorted(idx.Rank, &item) } - idx.Urls = append(idx.Urls, url) - log.Printf("added to feed: '%s' %v", url, tags) + idx.Urls = append(idx.Urls, struct{ Url, Category string }{url, category}) + log.Printf("added to feed: '%s' %v", url, category) } return err } @@ -50,24 +50,25 @@ func (idx *Index) Get(query Query) Result { } // loads (newsboat) file rss feeds into the index -func (idx *Index) Load(filename string) { +func (idx *Index) Load(filename string) error { if filename == "" { - return + return nil } file, err := os.Open(filename) if err != nil { log.Printf("can't open: %s", filename) - return + return err } scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() - if url, tags := parseLine(line); len(url) > 0 { - if err := idx.Add(url, tags); err != nil { + if url, category := parseLine(line); len(url) > 0 { + if err := idx.Add(url, category); err != nil { log.Printf("error adding url: '%s'", url) } } } + return nil } // servers all api endpoints for an index instance diff --git a/pkg/rss.go b/pkg/rss.go index 8a43e1a..2d8fc50 100644 --- a/pkg/rss.go +++ b/pkg/rss.go @@ -9,7 +9,7 @@ import ( // extracts keywords from title, used for querying based on keywords func (item *Item) Keywords() []string { re := regexp.MustCompile(`[a-zA-Z]+`) - words := re.FindAllString(strings.ToLower(item.Title), -1) + words := re.FindAllString(strings.ToLower(item.Description), -1) keywords := []string{} for _, w := range words { if len(w) >= 4 { @@ -25,7 +25,7 @@ func (item *Item) QueryMatch(query Query) bool { switch { case len(query.Urls) > 0 && !slices.Contains(query.Urls, item.parentFeed.Url): return false - case len(query.Tags) > 0 && !overlap(query.Tags, item.parentFeed.Tags): + case len(query.Category) > 0 && query.Category != item.parentFeed.Category: return false case len(query.Keywords) > 0 && !overlap(query.Keywords, item.Keywords()): return false diff --git a/pkg/types.go b/pkg/types.go index 8c4faf4..a7894b7 100644 --- a/pkg/types.go +++ b/pkg/types.go @@ -33,23 +33,22 @@ type Feed struct { Version string `xml:"version,attr"` Channel Channel `xml:"channel"` // generated - Tags []string - Url string + Category string + Url string } type Index struct { Rank []*Item - Urls []string - // Urls []struct { - // Url string - // Tags []string - // } + Urls []struct { + Url string + Category string + } } type Query struct { Urls []string - Tags []string Keywords []string + Category string Amount int } diff --git a/pkg/utils.go b/pkg/utils.go index ecb40fa..9e63cd8 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -41,18 +41,18 @@ func parsePubDate(s string) time.Time { return time.Time{} } -// parses a newsboat URL line and gets the url and tags -func parseLine(line string) (string, []string) { +// parses a newsboat URL line and gets the url and category +func parseLine(line string) (string, string) { parts := strings.Split(line, " ") url, err := url.Parse(parts[0]) - tags := "" + category := "" if err == nil { if len(parts) > 1 { - tags = parts[1] + category = parts[1] } - return url.String(), []string{tags} + return url.String(), category } - return "", []string{} // no valid URL found + return "", "" // no valid URL found } // inserts an item in sorted order. Also returns index it was inserted at. @@ -78,6 +78,16 @@ func getListParam(url *url.URL, param string) []string { return filteredParams } +func getStrParam(url *url.URL, param string) string { + params := url.Query().Get(param) + return strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + return r + } + return -1 + }, strings.ToLower(params)) +} + // gets integer param from url, and takes out bad values func getIntParam(url *url.URL, param string, fallback int) int { strValue := url.Query().Get(param) diff --git a/roxy.go b/roxy.go index e8b13c2..d86aab9 100644 --- a/roxy.go +++ b/roxy.go @@ -2,6 +2,7 @@ package main import ( "flag" + "log" pkg "github.com/TimoKats/roxy/pkg" ) @@ -13,6 +14,9 @@ func main() { flag.Parse() // start server idx := pkg.NewIndex() - idx.Load(*filename) + if err := idx.Load(*filename); err != nil { + log.Println("startup failed...") + return + } idx.Serve(":" + *port) } diff --git a/tests/regression_test.go b/tests/regression_test.go index 5bb4411..b88ef80 100644 --- a/tests/regression_test.go +++ b/tests/regression_test.go @@ -12,9 +12,7 @@ import ( func TestAdd(t *testing.T) { idx := pkg.NewIndex() url := "https://timokats.xyz/feed/website.xml" - tags := []string{"example", "test"} - err := idx.Add(url, tags) - if err != nil { + if err := idx.Add(url, "test"); err != nil { t.Errorf("Expected no error, got %v", err) } if len(idx.Rank) == 0 { @@ -25,9 +23,7 @@ func TestAdd(t *testing.T) { func TestGet(t *testing.T) { idx := pkg.NewIndex() url := "https://timokats.xyz/feed/website.xml" - tags := []string{"example", "test"} - err := idx.Add(url, tags) - if err != nil { + if err := idx.Add(url, "test3"); err != nil { t.Errorf("Expected no error, got %v", err) } query := pkg.Query{Amount: 10} @@ -40,9 +36,7 @@ func TestGet(t *testing.T) { func TestServe(t *testing.T) { idx := pkg.NewIndex() url := "https://timokats.xyz/feed/website.xml" - tags := []string{"example", "test"} - err := idx.Add(url, tags) - if err != nil { + if err := idx.Add(url, "test5"); err != nil { t.Errorf("Expected no error, got %v", err) } @@ -66,9 +60,7 @@ func TestServe(t *testing.T) { func TestClear(t *testing.T) { idx := pkg.NewIndex() url := "https://timokats.xyz/feed/website.xml" - tags := []string{"example", "test"} - err := idx.Add(url, tags) - if err != nil { + if err := idx.Add(url, ""); err != nil { t.Errorf("Expected no error, got %v", err) } idx.Clear() From 49c75fcd71ef2fd16afccd91a97b8cd7a0455f88 Mon Sep 17 00:00:00 2001 From: TimoKats Date: Fri, 24 Apr 2026 21:57:03 +0200 Subject: [PATCH 8/8] add json unmarshall --- pkg/api.go | 27 ++++++++++++++++---------- pkg/index.go | 10 ++++++++-- pkg/rss.go | 7 +++---- pkg/types.go | 42 ++++++++++++++++++++++++---------------- pkg/utils.go | 27 +++++++++++++++++++++++--- tests/regression_test.go | 2 +- 6 files changed, 78 insertions(+), 37 deletions(-) diff --git a/pkg/api.go b/pkg/api.go index 5085713..935eb5c 100644 --- a/pkg/api.go +++ b/pkg/api.go @@ -1,7 +1,6 @@ package pkg import ( - "encoding/xml" "log" "net/http" "strings" @@ -16,6 +15,14 @@ func (api Api) Ping() http.HandlerFunc { } } +func (api Api) Stats(idx *Index) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + data, header := marshall(idx.Urls, JSON) + w.Header().Set("Content-Type", header) + w.Write(data) //nolint:errcheck + } +} + // endpoint for adding rss feeds through api func (api Api) Add(idx *Index) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -38,18 +45,18 @@ func (api Api) Add(idx *Index) http.HandlerFunc { } // query the rss feeds using url parameters, returns xml in the body -func (api Api) Get(idx *Index) http.HandlerFunc { +func (api Api) Get(idx *Index, format Format) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { query := Query{ - Urls: getListParam(r.URL, "urls"), - Category: getStrParam(r.URL, "category"), - Keywords: getListParam(r.URL, "keywords"), - Amount: getIntParam(r.URL, "amount", 10), + Urls: getListParam(r.URL, "urls"), + Categories: getListParam(r.URL, "category"), + Keywords: getListParam(r.URL, "keywords"), + Amount: getIntParam(r.URL, "amount", 10), } result := idx.Get(query) - xmlData, _ := xml.MarshalIndent(result, "", "\t") - w.Header().Set("Content-Type", "application/xml") - w.Write(xmlData) //nolint:errcheck + data, header := marshall(result, format) + w.Header().Set("Content-Type", header) + w.Write(data) //nolint:errcheck } } @@ -58,7 +65,7 @@ func (api Api) Refresh(idx *Index) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { idx.Clear() for _, url := range idx.Urls { - log.Printf("refreshing: %s", url) + log.Printf("refreshing: %s", url.Url) err := idx.Add(url.Url, url.Category) if err != nil { http.Error(w, "fail: "+url.Url, http.StatusInternalServerError) diff --git a/pkg/index.go b/pkg/index.go index 12db0be..e67d564 100644 --- a/pkg/index.go +++ b/pkg/index.go @@ -23,7 +23,11 @@ func (idx *Index) Add(url string, category string) error { item.parentFeed = &feed idx.Rank = insertSorted(idx.Rank, &item) } - idx.Urls = append(idx.Urls, struct{ Url, Category string }{url, category}) + idx.Urls = append(idx.Urls, struct { + Url string + Category string + Size int + }{url, category, len(feed.Channel.Items)}) log.Printf("added to feed: '%s' %v", url, category) } return err @@ -76,8 +80,10 @@ func (idx *Index) Serve(port string) { api := Api{} log.Printf("serving on: http://localhost%s", port) http.HandleFunc("/", api.Ping()) + http.HandleFunc("/stats", api.Stats(idx)) http.HandleFunc("/add", api.Add(idx)) - http.HandleFunc("/get", api.Get(idx)) + http.HandleFunc("/xml", api.Get(idx, XML)) + http.HandleFunc("/json", api.Get(idx, JSON)) http.HandleFunc("/refresh", api.Refresh(idx)) log.Fatal(http.ListenAndServe(port, nil)) } diff --git a/pkg/rss.go b/pkg/rss.go index 2d8fc50..94660b4 100644 --- a/pkg/rss.go +++ b/pkg/rss.go @@ -2,7 +2,6 @@ package pkg import ( "regexp" - "slices" "strings" ) @@ -23,11 +22,11 @@ func (item *Item) Keywords() []string { // ps, King Terry said case/switch are devine, hence the choice func (item *Item) QueryMatch(query Query) bool { switch { - case len(query.Urls) > 0 && !slices.Contains(query.Urls, item.parentFeed.Url): + case !contains(query.Urls, item.parentFeed.Url): return false - case len(query.Category) > 0 && query.Category != item.parentFeed.Category: + case !contains(query.Categories, item.parentFeed.Category): return false - case len(query.Keywords) > 0 && !overlap(query.Keywords, item.Keywords()): + case !overlap(query.Keywords, item.Keywords()): return false default: return true diff --git a/pkg/types.go b/pkg/types.go index a7894b7..c94d9c1 100644 --- a/pkg/types.go +++ b/pkg/types.go @@ -5,24 +5,31 @@ import ( "time" ) +type Format string + +const ( + JSON Format = "json" + XML Format = "xml" +) + type Item struct { - Title string `xml:"title"` - Description string `xml:"description"` - Link string `xml:"link"` - Guid string `xml:"guid"` - PubDate string `xml:"pubDate"` + Title string `xml:"title" json:"title"` + Description string `xml:"description" json:"description"` + Link string `xml:"link" json:"link"` + Guid string `xml:"guid" json:"guid"` + PubDate string `xml:"pubDate" json:"pubDate"` // generated timestamp time.Time parentFeed *Feed } type Channel struct { - Title string `xml:"title"` - Description string `xml:"description"` - Link string `xml:"link"` + Title string `xml:"title" json:"title"` + Description string `xml:"description" json:"description"` + Link string `xml:"link" json:"link"` Items []Item `xml:"item"` - PubDate string `xml:"pubDate"` - Category []string `xml:"category"` + PubDate string `xml:"pubDate" json:"pubDate"` + Category []string `xml:"category" json:"category"` Generator string `xml:"generator"` // generated timestamp time.Time @@ -42,18 +49,19 @@ type Index struct { Urls []struct { Url string Category string + Size int } } type Query struct { - Urls []string - Keywords []string - Category string - Amount int + Urls []string + Keywords []string + Categories []string + Amount int } type Result struct { - XMLName xml.Name `xml:"rss"` - Version string `xml:"version,attr"` - Items []*Item `xml:"channel>item"` + XMLName xml.Name `xml:"rss" json:"-"` + Version string `xml:"version,attr" json:"-"` + Items []*Item `xml:"channel>item" json:"items"` } diff --git a/pkg/utils.go b/pkg/utils.go index 9e63cd8..377d2f1 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -1,6 +1,8 @@ package pkg import ( + "encoding/json" + "encoding/xml" "net/url" "slices" "sort" @@ -9,10 +11,21 @@ import ( "time" ) -// check if two lists have any overlap. Useful for querying. +// calls json/xml marshall function to format result object +func marshall(data any, format Format) ([]byte, string) { + var result []byte + if format == JSON { + result, _ = json.Marshal(data) //nolint:errcheck + return result, "application/json" + } + result, _ = xml.MarshalIndent(data, "", "\t") //nolint:errcheck + return result, "application/xml" +} + +// check if two lists have any overlap. Useful for querying func overlap[Type comparable](a []Type, b []Type) bool { if len(a) == 0 || len(b) == 0 { - return false + return true } for _, aItem := range a { if slices.Contains(b, aItem) { @@ -22,6 +35,14 @@ func overlap[Type comparable](a []Type, b []Type) bool { return false } +// test if item exists in a list of comparable items +func contains[Type comparable](list []Type, item Type) bool { + if len(list) == 0 { + return true + } + return slices.Contains(list, item) +} + // tries all mentally sane rss datetime formats and returns time object func parsePubDate(s string) time.Time { var rssDateFormats = []string{ @@ -50,7 +71,7 @@ func parseLine(line string) (string, string) { if len(parts) > 1 { category = parts[1] } - return url.String(), category + return url.String(), strings.ReplaceAll(category, `"`, "") } return "", "" // no valid URL found } diff --git a/tests/regression_test.go b/tests/regression_test.go index b88ef80..7911988 100644 --- a/tests/regression_test.go +++ b/tests/regression_test.go @@ -42,7 +42,7 @@ func TestServe(t *testing.T) { go idx.Serve(":8080") time.Sleep(2 * time.Second) - resp, err := http.Get("http://localhost:8080/get?amount=10") + resp, err := http.Get("http://localhost:8080/xml?amount=10") if err != nil { t.Errorf("Expected no error, got %v", err) }