feat(filter): add HTTP filter plugin

pkg/filter/plugins/http_plugin.go

@@ -0,0 +1,125 @@
+package plugins
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"sync/atomic"
+
+	"github.com/ncarlier/feedpushr/v3/pkg/common"
+	"github.com/ncarlier/feedpushr/v3/pkg/expr"
+	"github.com/ncarlier/feedpushr/v3/pkg/format"
+	"github.com/ncarlier/feedpushr/v3/pkg/model"
+)
+
+var httpSpec = model.Spec{
+	Name: "http",
+	Desc: `
+This filter will send the article as JSON object to a HTTP endpoint (POST).
+
+HTTP endpoint must return same [JSON structure](https://github.com/ncarlier/feedpushr#output-format).
+If succeeded, the response is merged with the current article.
+`,
+	PropsSpec: []model.PropSpec{
+		{
+			Name: "url",
+			Desc: "Target URL",
+			Type: model.URL,
+		},
+	},
+}
+
+// HTTPFilterPlugin is the http filter plugin
+type HTTPFilterPlugin struct{}
+
+// Spec returns plugin spec
+func (p *HTTPFilterPlugin) Spec() model.Spec {
+	return httpSpec
+}
+
+// Build creates http filter
+func (p *HTTPFilterPlugin) Build(def *model.FilterDef) (model.Filter, error) {
+	condition, err := expr.NewConditionalExpression(def.Condition)
+	if err != nil {
+		return nil, err
+	}
+
+	u, ok := def.Props["url"]
+	if !ok {
+		return nil, fmt.Errorf("missing URL property")
+	}
+	targetURL, err := url.ParseRequestURI(fmt.Sprintf("%v", u))
+	if err != nil {
+		return nil, fmt.Errorf("invalid URL property: %s", err.Error())
+	}
+
+	definition := *def
+	definition.Spec = httpSpec
+	definition.Props["url"] = targetURL.String()
+
+	return &HTTPFilter{
+		definition: definition,
+		condition:  condition,
+		targetURL:  targetURL.String(),
+		formatter:  format.NewJSONFormatter(),
+	}, nil
+}
+
+// HTTPFilter is a filter that try to http the original article content
+type HTTPFilter struct {
+	definition model.FilterDef
+	condition  *expr.ConditionalExpression
+	targetURL  string
+	formatter  format.Formatter
+}
+
+// DoFilter applies filter on the article
+func (f *HTTPFilter) DoFilter(article *model.Article) (bool, error) {
+	b, err := f.formatter.Format(article)
+	if err != nil {
+		atomic.AddUint64(&f.definition.NbError, 1)
+		return false, err
+	}
+
+	req, err := http.NewRequest("POST", f.targetURL, b)
+	if err != nil {
+		atomic.AddUint64(&f.definition.NbError, 1)
+		return false, err
+	}
+	req.Header.Set("User-Agent", common.UserAgent)
+	req.Header.Set("Content-Type", common.ContentTypeJSON)
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		atomic.AddUint64(&f.definition.NbError, 1)
+		return false, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 300 {
+		atomic.AddUint64(&f.definition.NbError, 1)
+		return false, fmt.Errorf("bad status code: %d", resp.StatusCode)
+	}
+
+	var returnedArticle model.Article
+	if err := json.NewDecoder(resp.Body).Decode(&returnedArticle); err != nil {
+		atomic.AddUint64(&f.definition.NbError, 1)
+		return false, fmt.Errorf("invalid JSON payload: %s", err.Error())
+	}
+
+	// Merge article with the response
+	article.Merge(returnedArticle)
+
+	atomic.AddUint64(&f.definition.NbSuccess, 1)
+	return true, nil
+}
+
+// Match test if article matches filter condition
+func (f *HTTPFilter) Match(article *model.Article) bool {
+	return f.condition.Match(article)
+}
+
+// GetDef return filter definition
+func (f *HTTPFilter) GetDef() model.FilterDef {
+	return f.definition
+}

pkg/filter/plugins/plugins.go

@@ -8,9 +8,11 @@ func GetBuiltinFilterPlugins() map[string]model.FilterPlugin {
 	titleFilterPlugin := &TitleFilterPlugin{}
 	minifyFilterPlugin := &MinifyFilterPlugin{}
 	fetchFilterPlugin := &FetchFilterPlugin{}
+	httpFilterPlugin := &HTTPFilterPlugin{}
 
 	plugins[titleFilterPlugin.Spec().Name] = titleFilterPlugin
 	plugins[minifyFilterPlugin.Spec().Name] = minifyFilterPlugin
 	plugins[fetchFilterPlugin.Spec().Name] = fetchFilterPlugin
+	plugins[httpFilterPlugin.Spec().Name] = httpFilterPlugin
 	return plugins
 }

pkg/filter/test/http_filter_test.go

@@ -0,0 +1,27 @@
+package test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/ncarlier/feedpushr/v3/pkg/model"
+)
+
+func TestHTTPFilter(t *testing.T) {
+	chain := buildChainFilter(t, "http://?url=https://run.mocky.io/v3/64073093-4e20-412d-8e0a-a6a1e23c01bd")
+
+	article := &model.Article{
+		Title:   "hello world",
+		Content: "<p>this should be replaced</p>",
+		Text:    "this should be kept",
+		Meta:    make(map[string]interface{}),
+	}
+	article.Meta["foo"] = "bar"
+	err := chain.Apply(article)
+	assert.Nil(t, err)
+	assert.Equal(t, "A mock", article.Title)
+	assert.Equal(t, "<p>with a fake content</p>", article.Content)
+	assert.Equal(t, "this should be kept", article.Text)
+	assert.Equal(t, "bar", article.Meta["foo"])
+}

pkg/model/article.go

@@ -28,6 +28,28 @@ func (a *Article) String() string {
 	return string(result)
 }
 
+// Merge an article with an other
+func (a *Article) Merge(other Article) {
+	if other.Link != "" {
+		a.Link = other.Link
+	}
+	if other.Title != "" {
+		a.Title = other.Title
+	}
+	if other.Content != "" {
+		a.Content = other.Content
+	}
+	if other.Text != "" {
+		a.Text = other.Text
+	}
+	if len(other.Tags) > 0 {
+		a.Tags = other.Tags
+	}
+	for k, v := range other.Meta {
+		a.Meta[k] = v
+	}
+}
+
 // RefDate get article reference date (published or updated date)
 func (a *Article) RefDate() *time.Time {
 	var date *time.Time