diff --git a/pkg/filter/plugins/http_plugin.go b/pkg/filter/plugins/http_plugin.go new file mode 100644 index 0000000..b46c680 --- /dev/null +++ b/pkg/filter/plugins/http_plugin.go @@ -0,0 +1,125 @@ +package plugins + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "sync/atomic" + + "github.com/ncarlier/feedpushr/v3/pkg/common" + "github.com/ncarlier/feedpushr/v3/pkg/expr" + "github.com/ncarlier/feedpushr/v3/pkg/format" + "github.com/ncarlier/feedpushr/v3/pkg/model" +) + +var httpSpec = model.Spec{ + Name: "http", + Desc: ` +This filter will send the article as JSON object to a HTTP endpoint (POST). + +HTTP endpoint must return same [JSON structure](https://github.com/ncarlier/feedpushr#output-format). +If succeeded, the response is merged with the current article. +`, + PropsSpec: []model.PropSpec{ + { + Name: "url", + Desc: "Target URL", + Type: model.URL, + }, + }, +} + +// HTTPFilterPlugin is the http filter plugin +type HTTPFilterPlugin struct{} + +// Spec returns plugin spec +func (p *HTTPFilterPlugin) Spec() model.Spec { + return httpSpec +} + +// Build creates http filter +func (p *HTTPFilterPlugin) Build(def *model.FilterDef) (model.Filter, error) { + condition, err := expr.NewConditionalExpression(def.Condition) + if err != nil { + return nil, err + } + + u, ok := def.Props["url"] + if !ok { + return nil, fmt.Errorf("missing URL property") + } + targetURL, err := url.ParseRequestURI(fmt.Sprintf("%v", u)) + if err != nil { + return nil, fmt.Errorf("invalid URL property: %s", err.Error()) + } + + definition := *def + definition.Spec = httpSpec + definition.Props["url"] = targetURL.String() + + return &HTTPFilter{ + definition: definition, + condition: condition, + targetURL: targetURL.String(), + formatter: format.NewJSONFormatter(), + }, nil +} + +// HTTPFilter is a filter that try to http the original article content +type HTTPFilter struct { + definition model.FilterDef + condition *expr.ConditionalExpression + targetURL string + formatter format.Formatter +} + +// DoFilter applies filter on the article +func (f *HTTPFilter) DoFilter(article *model.Article) (bool, error) { + b, err := f.formatter.Format(article) + if err != nil { + atomic.AddUint64(&f.definition.NbError, 1) + return false, err + } + + req, err := http.NewRequest("POST", f.targetURL, b) + if err != nil { + atomic.AddUint64(&f.definition.NbError, 1) + return false, err + } + req.Header.Set("User-Agent", common.UserAgent) + req.Header.Set("Content-Type", common.ContentTypeJSON) + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + atomic.AddUint64(&f.definition.NbError, 1) + return false, err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + atomic.AddUint64(&f.definition.NbError, 1) + return false, fmt.Errorf("bad status code: %d", resp.StatusCode) + } + + var returnedArticle model.Article + if err := json.NewDecoder(resp.Body).Decode(&returnedArticle); err != nil { + atomic.AddUint64(&f.definition.NbError, 1) + return false, fmt.Errorf("invalid JSON payload: %s", err.Error()) + } + + // Merge article with the response + article.Merge(returnedArticle) + + atomic.AddUint64(&f.definition.NbSuccess, 1) + return true, nil +} + +// Match test if article matches filter condition +func (f *HTTPFilter) Match(article *model.Article) bool { + return f.condition.Match(article) +} + +// GetDef return filter definition +func (f *HTTPFilter) GetDef() model.FilterDef { + return f.definition +} diff --git a/pkg/filter/plugins/plugins.go b/pkg/filter/plugins/plugins.go index a54852a..33d2d62 100644 --- a/pkg/filter/plugins/plugins.go +++ b/pkg/filter/plugins/plugins.go @@ -8,9 +8,11 @@ func GetBuiltinFilterPlugins() map[string]model.FilterPlugin { titleFilterPlugin := &TitleFilterPlugin{} minifyFilterPlugin := &MinifyFilterPlugin{} fetchFilterPlugin := &FetchFilterPlugin{} + httpFilterPlugin := &HTTPFilterPlugin{} plugins[titleFilterPlugin.Spec().Name] = titleFilterPlugin plugins[minifyFilterPlugin.Spec().Name] = minifyFilterPlugin plugins[fetchFilterPlugin.Spec().Name] = fetchFilterPlugin + plugins[httpFilterPlugin.Spec().Name] = httpFilterPlugin return plugins } diff --git a/pkg/filter/test/http_filter_test.go b/pkg/filter/test/http_filter_test.go new file mode 100644 index 0000000..1549ece --- /dev/null +++ b/pkg/filter/test/http_filter_test.go @@ -0,0 +1,27 @@ +package test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/ncarlier/feedpushr/v3/pkg/model" +) + +func TestHTTPFilter(t *testing.T) { + chain := buildChainFilter(t, "http://?url=https://run.mocky.io/v3/64073093-4e20-412d-8e0a-a6a1e23c01bd") + + article := &model.Article{ + Title: "hello world", + Content: "

this should be replaced

", + Text: "this should be kept", + Meta: make(map[string]interface{}), + } + article.Meta["foo"] = "bar" + err := chain.Apply(article) + assert.Nil(t, err) + assert.Equal(t, "A mock", article.Title) + assert.Equal(t, "

with a fake content

", article.Content) + assert.Equal(t, "this should be kept", article.Text) + assert.Equal(t, "bar", article.Meta["foo"]) +} diff --git a/pkg/model/article.go b/pkg/model/article.go index 8976542..74b20c1 100644 --- a/pkg/model/article.go +++ b/pkg/model/article.go @@ -28,6 +28,28 @@ func (a *Article) String() string { return string(result) } +// Merge an article with an other +func (a *Article) Merge(other Article) { + if other.Link != "" { + a.Link = other.Link + } + if other.Title != "" { + a.Title = other.Title + } + if other.Content != "" { + a.Content = other.Content + } + if other.Text != "" { + a.Text = other.Text + } + if len(other.Tags) > 0 { + a.Tags = other.Tags + } + for k, v := range other.Meta { + a.Meta[k] = v + } +} + // RefDate get article reference date (published or updated date) func (a *Article) RefDate() *time.Time { var date *time.Time