Add more metadata to plugins

api/http.go

@@ -6,12 +6,25 @@ import (
 	"net/http"
 
 	"github.com/go-chi/chi"
+	"github.com/go-chi/cors"
 )
 
 // NewHandler initializes a new feed router
 func NewHandler(s service) *chi.Mux {
 	r := chi.NewRouter()
 
+	cors := cors.New(cors.Options{
+		// AllowedOrigins: []string{"https://foo.com"}, // Use this to allow specific origin hosts
+		AllowedOrigins: []string{"*"},
+		// AllowOriginFunc:  func(r *http.Request, origin string) bool { return true },
+		AllowedMethods:   []string{"GET", "POST", "OPTIONS"},
+		AllowedHeaders:   []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"},
+		ExposedHeaders:   []string{"Link"},
+		AllowCredentials: true,
+		MaxAge:           300, // Maximum value not ignored by any of major browsers
+	})
+	r.Use(cors.Handler)
+
 	// Public routes
 	r.Group(func(r chi.Router) {
 		r.Get("/list", getPluginListHandler(s))

api/service.go

@@ -43,11 +43,17 @@ func (s *service) ServeFeed(format string, plugin string) (string, error) {
 	return feed, nil
 }
 
-func (s *service) ListFeeds() []string {
-	p := s.pluginRepository.All()
-	var plugins []string
-	for _, p := range p {
-		plugins = append(plugins, p.String())
+func (s *service) ListFeeds() []plugin.PluginMetadata {
+	var pp []plugin.PluginMetadata
+	for _, p := range s.pluginRepository.All() {
+		pp = append(pp, plugin.PluginMetadata{
+			Name:          p.Info().Name,
+			Description:   p.Info().Description,
+			TechnicalName: p.Info().TechnicalName,
+			Image:         p.Info().Image,
+			Author:        p.Info().Author,
+			SourceURL:     p.Info().SourceURL,
+		})
 	}
-	return plugins
+	return pp
 }

main.go

@@ -30,6 +30,7 @@ func main() {
 		CacheExpiredPurge int    `env:"CACHE_EXPIRED_PURGE" envDefault:"60"`
 		Environment       string `env:"ENVIRONMENT" envDefault:"develop"`
 		Port              int    `env:"PORT" envDefault:"8080"`
+		APIHostname       string `env:"API_HOSTNAME" envDefault:"http://localhost"`
 	}
 	err := env.Parse(&config)
 	if err != nil {
@@ -77,7 +78,7 @@ func main() {
 	r.Handle("/metrics", promhttp.Handler())
 	// TODO(dewey): Switch to promhttp middleware instead of this deprecated one
 	r.Mount("/feed", prometheus.InstrumentHandler("feed", api.NewHandler(*apiService)))
-	l.Log("msg", fmt.Sprintf("feedbridge listening on :%d", config.Port))
+	l.Log("msg", fmt.Sprintf("feedbridge listening on %s:%d", config.APIHostname, config.Port))
 	err = http.ListenAndServe(fmt.Sprintf(":%d", config.Port), r)
 	if err != nil {
 		panic(err)

plugin/mem_repository.go

@@ -28,11 +28,11 @@ func (mr *MemRepo) Install(p Plugin) {
 	mr.Lock()
 	defer mr.Unlock()
 
-	if _, ok := mr.plugins[p.String()]; ok {
-		panic(fmt.Sprintf("plugin with name '%s' already exists", p.String()))
+	if _, ok := mr.plugins[p.Info().TechnicalName]; ok {
+		panic(fmt.Sprintf("plugin with name '%s' already exists", p.Info().TechnicalName))
 	}
 
-	mr.plugins[p.String()] = p
+	mr.plugins[p.Info().TechnicalName] = p
 }
 
 // Find finds a plugin by name

plugin/repository.go

@@ -14,5 +14,14 @@ type Repository interface {
 // Plugin is the interface that a scrape plugin has to implement
 type Plugin interface {
 	Run() (*feeds.Feed, error)
-	String() string
+	Info() PluginMetadata
+}
+
+type PluginMetadata struct {
+	TechnicalName string
+	Name          string
+	Description   string
+	Author        string
+	Image         string
+	SourceURL     string
 }

plugins/scmp/scmp.go

@@ -3,11 +3,11 @@ package scmp
 import (
 	"fmt"
 	"net/http"
-	"net/url"
-	"strconv"
 	"time"
 
+	pm "github.com/dewey/feedbridge/plugin"
 	"github.com/dewey/feedbridge/scrape"
+
 	"github.com/go-kit/kit/log"
 
 	"github.com/PuerkitoBio/goquery"
@@ -35,8 +35,15 @@ func NewPlugin(l log.Logger, c *http.Client) *plugin {
 	}
 }
 
-func (p *plugin) String() string {
-	return "scmp"
+func (p *plugin) Info() pm.PluginMetadata {
+	return pm.PluginMetadata{
+		TechnicalName: "scmp",
+		Name:          "The South China Morning Post: Infographics",
+		Description:   `Scraping the Infographics section of South China Morning Post as there's no RSS feed available. The Infographics cover various topics like Politics, Lifestyle, Science and Economy.`,
+		Author:        "Philipp",
+		Image:         "https://i.imgur.com/pPVoXVh.png",
+		SourceURL:     "https://www.scmp.com/topics/infographics-asia",
+	}
 }
 
 // Run runs the main checker function of the plugin
@@ -64,33 +71,33 @@ func (p *plugin) Run() (*feeds.Feed, error) {
 		feedItems = append(feedItems, items...)
 
 		// Create tasks for pagination
-		var subTask []scrape.Task
-		for i := 1; i < 5; i++ {
-			u, err := url.Parse(r.URL)
-			if err != nil {
-				p.l.Log("err", err)
-				continue
-			}
-			q := u.Query()
-			q.Add("page", strconv.Itoa(i))
-			u.RawQuery = q.Encode()
-			subTask = append(subTask, scrape.Task{
-				URL: u.String(),
-			})
-		}
+		// var subTask []scrape.Task
+		// for i := 1; i < 5; i++ {
+		// 	u, err := url.Parse(r.URL)
+		// 	if err != nil {
+		// 		p.l.Log("err", err)
+		// 		continue
+		// 	}
+		// 	q := u.Query()
+		// 	q.Add("page", strconv.Itoa(i))
+		// 	u.RawQuery = q.Encode()
+		// 	subTask = append(subTask, scrape.Task{
+		// 		URL: u.String(),
+		// 	})
+		// }
 
 		// Get all items from other pages
-		result, err := scrape.URLToDocument(p.c, subTask)
-		if err != nil {
-			return nil, err
-		}
-		for _, r := range result {
-			items, err := p.listHandler(&r.Document)
-			if err != nil {
-				p.l.Log("err", err)
-			}
-			feedItems = append(feedItems, items...)
-		}
+		// result, err := scrape.URLToDocument(p.c, subTask)
+		// if err != nil {
+		// 	return nil, err
+		// }
+		// for _, r := range result {
+		// 	items, err := p.listHandler(&r.Document)
+		// 	if err != nil {
+		// 		p.l.Log("err", err)
+		// 	}
+		// 	feedItems = append(feedItems, items...)
+		// }
 	}
 	p.f.Items = feedItems
 	return p.f, nil
@@ -133,14 +140,19 @@ func (p *plugin) listHandler(doc *goquery.Document) ([]*feeds.Item, error) {
 		times := s.Find("time.updated")
 		val, exists = times.Attr("content")
 		if exists {
+			fmt.Println(val)
 			t, err := time.Parse("2006-01-02T15:04:05-07:00", val)
 			if err == nil {
+				fmt.Println(t)
 				item.Updated = t
+				item.Created = t
 			} else {
 				p.l.Log("err", err)
 			}
 		}
 
+		fmt.Println(item)
+
 		feedItems = append(feedItems, item)
 	})
 	return feedItems, nil

runner/runner.go

@@ -76,17 +76,17 @@ func (r *Runner) Start() {
 			go func(cp plugin.Plugin) {
 				defer wg.Done()
 				start := time.Now()
-				level.Info(log.With(r.l, "plugin", cp.String())).Log("msg", "scrape started")
+				level.Info(log.With(r.l, "plugin", cp.Info().TechnicalName)).Log("msg", "scrape started")
 				ss, err := r.runPlugin(cp)
 				if err != nil {
 					level.Error(r.l).Log("err", err)
 					return
 				}
 
 				duration := time.Since(start)
-				scrapesDurationHistogram.WithLabelValues(cp.String()).Observe(duration.Seconds())
-				pluginItemsScraped.WithLabelValues(cp.String()).Set(float64(ss.Items))
-				level.Info(log.With(r.l, "plugin", cp.String())).Log("msg", "scrape finished", "feed_items", ss.Items)
+				scrapesDurationHistogram.WithLabelValues(cp.Info().TechnicalName).Observe(duration.Seconds())
+				pluginItemsScraped.WithLabelValues(cp.Info().TechnicalName).Set(float64(ss.Items))
+				level.Info(log.With(r.l, "plugin", cp.Info().TechnicalName)).Log("msg", "scrape finished", "feed_items", ss.Items)
 			}(cp)
 		}
 		wg.Wait()
@@ -105,18 +105,18 @@ func (r *Runner) runPlugin(cp plugin.Plugin) (scrape.Statistic, error) {
 	if err != nil {
 		return scrape.Statistic{}, err
 	}
-	r.StorageRepository.Save(fmt.Sprintf("rss_%s", cp.String()), rss)
+	r.StorageRepository.Save(fmt.Sprintf("rss_%s", cp.Info().TechnicalName), rss)
 
 	atom, err := f.ToAtom()
 	if err != nil {
 		return scrape.Statistic{}, err
 	}
-	r.StorageRepository.Save(fmt.Sprintf("atom_%s", cp.String()), atom)
+	r.StorageRepository.Save(fmt.Sprintf("atom_%s", cp.Info().TechnicalName), atom)
 
 	json, err := f.ToJSON()
 	if err != nil {
 		return scrape.Statistic{}, err
 	}
-	r.StorageRepository.Save(fmt.Sprintf("json_%s", cp.String()), json)
+	r.StorageRepository.Save(fmt.Sprintf("json_%s", cp.Info().TechnicalName), json)
 	return scrape.Statistic{Items: len(f.Items)}, nil
 }