Merge pull request #1956 from traPtitech/add-useragent-bot

add user-agent bot in ParseMetaForURL
traPtitech · Sep 5, 2023 · 08216f6 · 08216f6
2 parents 83dfecc + e2ee810
commit 08216f6
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 2 deletions.
diff --git a/service/ogp/parser/domain.go b/service/ogp/parser/domain.go
@@ -12,7 +12,8 @@ var client = http.Client{
 	Timeout: 5 * time.Second,
 }
 
-const userAgent = "traq-ogp-fetcher; contact: github.com/traPtitech/traQ"
+// X(Twitter)のOGPを取得するのにuserAgentの中にbotという文字列が入っている必要がある
+const userAgent = "traq-ogp-fetcher-bot; contact: github.com/traPtitech/traQ"
 
 func FetchSpecialDomainInfo(url *url.URL) (og *opengraph.OpenGraph, meta *DefaultPageMeta, isSpecialDomain bool, err error) {
 	switch url.Host {

diff --git a/service/ogp/parser/parser.go b/service/ogp/parser/parser.go
@@ -34,7 +34,15 @@ func ParseMetaForURL(url *url.URL) (*opengraph.OpenGraph, *DefaultPageMeta, erro
 	client := http.Client{
 		Timeout: 5 * time.Second,
 	}
-	resp, err := client.Get(url.String())
+
+	req, err := http.NewRequest("GET", url.String(), nil)
+	if err != nil {
+		return nil, nil, ErrNetwork
+	}
+
+	req.Header.Add("user-agent", userAgent)
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, nil, ErrNetwork
 	}

diff --git a/service/ogp/parser/parser_test.go b/service/ogp/parser/parser_test.go
@@ -1,9 +1,12 @@
 package parser
 
 import (
+	"fmt"
+	"net/url"
 	"strings"
 	"testing"
 
+	"github.com/dyatlov/go-opengraph/opengraph"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/net/html"
 )
@@ -122,3 +125,42 @@ func TestExtractTitleFromNode(t *testing.T) {
 		assert.Equal(t, "", result)
 	})
 }
+
+func TestFetchTwitterOGP(t *testing.T) {
+	tests := []struct {
+		name    string
+		url     string
+		want    func(t *testing.T, res *opengraph.OpenGraph)
+		wantErr assert.ErrorAssertionFunc
+	}{
+		{
+			name: "success",
+			url:  "https://twitter.com/traPtitech/status/1690533645923287040",
+			want: func(t *testing.T, res *opengraph.OpenGraph) {
+				assert.Equal(t, "設営完了しました！\n西え-33aにてお待ちしています！\n#C102", res.Description)
+			},
+			wantErr: assert.NoError,
+		},
+		{
+			name: "not found",
+			url:  "https://twitter.com/traPtitech/status/1690533645923287041",
+			want: func(t *testing.T, res *opengraph.OpenGraph) {
+				assert.Equal(t, "", res.Description)
+			},
+			wantErr: assert.NoError,
+		},
+	}
+	t.Parallel()
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			u, _ := url.Parse(tt.url)
+			got, _, err := ParseMetaForURL(u)
+			if !tt.wantErr(t, err, fmt.Sprintf("ParseMetaForURL(%v)", tt.url)) {
+				return
+			}
+			tt.want(t, got)
+		})
+	}
+}