From f5d337b53703c67dc9e1b17315bc384efeb069a6 Mon Sep 17 00:00:00 2001 From: Johnny Debt <38006769+johnny-debt@users.noreply.github.com> Date: Wed, 15 Aug 2018 15:10:28 +0300 Subject: [PATCH] Issue #4 investigation --- instascrap.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++ instascrap_test.go | 17 +++++++++++++ mapping_test.go | 3 ++- media.go | 6 ++--- media_test.go | 27 +++++++++++--------- utils.go | 27 -------------------- utils_test.go | 33 +++++++++++++----------- 7 files changed, 119 insertions(+), 57 deletions(-) create mode 100644 instascrap.go delete mode 100644 utils.go diff --git a/instascrap.go b/instascrap.go new file mode 100644 index 0000000..6d7353a --- /dev/null +++ b/instascrap.go @@ -0,0 +1,63 @@ +package instascrap + +import ( + "errors" + "io" + "net/http" +) + +// HTTPClientProvider is a factor +type HTTPClientProvider func() *http.Client + +// Instascrap is main object which provides access to other functions +type Instascrap struct { + httpClientProvider HTTPClientProvider +} + +// NewInstascrap instantiate Instascrap structure +func NewInstascrap(httpClientProvider HTTPClientProvider) Instascrap { + instascrap := Instascrap{} + + if httpClientProvider != nil { + instascrap.httpClientProvider = httpClientProvider + } else { + instascrap.httpClientProvider = defaultHTTPClientProvider + } + + return instascrap +} + +// Provides default HTTP client +func defaultHTTPClientProvider() *http.Client { + return &http.Client{} +} + +func (instascrap *Instascrap) getDataFromURL(URL string, reader func(r io.Reader) ([]byte, error)) ([]byte, error) { + // create a request + req, err := http.NewRequest("GET", URL, nil) + if err != nil { + return nil, err + } + // use the http client to fetch the page + httpClient := instascrap.httpClientProvider() + if httpClient == nil { + return nil, errors.New("HTTP Client not available") + } + resp, err := httpClient.Do(req) + + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + return nil, errors.New("statusCode != 200") + } + defer resp.Body.Close() + + body, err := reader(resp.Body) + if err != nil { + return nil, err + } + + return body, nil +} diff --git a/instascrap_test.go b/instascrap_test.go index 860bbd1..54781ec 100644 --- a/instascrap_test.go +++ b/instascrap_test.go @@ -2,10 +2,27 @@ package instascrap import ( "io/ioutil" + "net/http" "os" "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" ) +func TestInstascrapDefaultCreation(t *testing.T) { + NewInstascrap(nil) +} + +func getEmptyHTTPClient() *http.Client { + return nil +} +func TestCustomHTTPClientProvider(t *testing.T) { + instascrap := NewInstascrap(getEmptyHTTPClient) + _, error := instascrap.getDataFromURL("http://google.com", nil) + assert.Error(t, error) +} + func getTestDataPath() string { pwd, _ := os.Getwd() return filepath.Join(pwd, "test-data") diff --git a/mapping_test.go b/mapping_test.go index 097b553..1b16ca5 100644 --- a/mapping_test.go +++ b/mapping_test.go @@ -1,8 +1,9 @@ package instascrap import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestGetMediasFromHashtagPage(t *testing.T) { diff --git a/media.go b/media.go index cd39c9b..b04350f 100644 --- a/media.go +++ b/media.go @@ -19,11 +19,11 @@ type Media struct { IsAdvertising bool } -// Returns latest medias from the hashtag page -func GetHashtagMedia(tag string) ([]Media, error) { +// GetHashtagMedia returns latest medias from the hashtag page +func (instascrap *Instascrap) GetHashtagMedia(tag string) ([]Media, error) { var medias []Media url := fmt.Sprintf(hashtagMediasURL, tag, "") - jsonBody, err := getDataFromURL(url, ioutil.ReadAll) + jsonBody, err := instascrap.getDataFromURL(url, ioutil.ReadAll) if err != nil { return nil, err } diff --git a/media_test.go b/media_test.go index 453e584..e2057e2 100644 --- a/media_test.go +++ b/media_test.go @@ -2,29 +2,31 @@ package instascrap import ( "fmt" - "github.com/stretchr/testify/assert" - "gopkg.in/h2non/gock.v1" "testing" + + "github.com/stretchr/testify/assert" + gock "gopkg.in/h2non/gock.v1" ) // Ensures that this method returns exactly response body func TestGetHashtagMediaSuccessful(t *testing.T) { defer gock.Off() hashtag := "something" - maxId := "" + maxID := "" apiEndpoint := "https://www.instagram.com" - apiUri := fmt.Sprintf("explore/tags/%s", hashtag) - params := map[string]string{"__a": "1", "max_id": maxId} + apiURI := fmt.Sprintf("explore/tags/%s", hashtag) + params := map[string]string{"__a": "1", "max_id": maxID} json := ReadTestDataFile("test-01-get-medias-from-hashtag-page.json") gock.New(apiEndpoint). - Get(apiUri). + Get(apiURI). MatchParams(params). Reply(200). JSON(json) - medias, err := GetHashtagMedia(hashtag) + instascrap := NewInstascrap(nil) + medias, err := instascrap.GetHashtagMedia(hashtag) assert.NoError(t, err) assert.Len(t, medias, 63) @@ -35,18 +37,19 @@ func TestGetHashtagMediaSuccessful(t *testing.T) { func TestGetHashtagMediaJSONRetrievingError(t *testing.T) { defer gock.Off() hashtag := "something" - maxId := "" + maxID := "" apiEndpoint := "https://www.instagram.com" - apiUri := fmt.Sprintf("explore/tags/%s", hashtag) - params := map[string]string{"__a": "1", "max_id": maxId} + apiURI := fmt.Sprintf("explore/tags/%s", hashtag) + params := map[string]string{"__a": "1", "max_id": maxID} gock.New(apiEndpoint). - Get(apiUri). + Get(apiURI). MatchParams(params). Reply(201). JSON("") - medias, err := GetHashtagMedia(hashtag) + instascrap := NewInstascrap(nil) + medias, err := instascrap.GetHashtagMedia(hashtag) assert.Error(t, err) assert.Nil(t, medias) diff --git a/utils.go b/utils.go deleted file mode 100644 index 8141c9e..0000000 --- a/utils.go +++ /dev/null @@ -1,27 +0,0 @@ -package instascrap - -import ( - "errors" - "io" - "net/http" -) - -func getDataFromURL(url string, reader func(r io.Reader) ([]byte, error)) ([]byte, error) { - resp, err := http.Get(url) - - if err != nil { - return nil, err - } - - if resp.StatusCode != 200 { - return nil, errors.New("statusCode != 200") - } - defer resp.Body.Close() - - body, err := reader(resp.Body) - if err != nil { - return nil, err - } - - return body, nil -} diff --git a/utils_test.go b/utils_test.go index 941537a..523550a 100644 --- a/utils_test.go +++ b/utils_test.go @@ -2,27 +2,29 @@ package instascrap import ( "errors" - "github.com/stretchr/testify/assert" - "gopkg.in/h2non/gock.v1" "io" "io/ioutil" "testing" + + "github.com/stretchr/testify/assert" + gock "gopkg.in/h2non/gock.v1" ) // Ensures that this method returns exactly response body func TestGetDataFromUrlSuccessful(t *testing.T) { defer gock.Off() - apiUrl := "https://example.com" + apiURL := "https://example.com" apiPath := "status" expectedResponse := "anything" - gock.New(apiUrl). + gock.New(apiURL). Get(apiPath). Reply(200). BodyString(expectedResponse) - actualResponse, err := getDataFromURL(apiUrl+"/"+apiPath, ioutil.ReadAll) + instascrap := NewInstascrap(nil) + actualResponse, err := instascrap.getDataFromURL(apiURL+"/"+apiPath, ioutil.ReadAll) assert.Equal(t, []byte(expectedResponse), actualResponse) assert.NoError(t, err) @@ -32,15 +34,16 @@ func TestGetDataFromUrlSuccessful(t *testing.T) { func TestGetDataFromUrlError(t *testing.T) { defer gock.Off() - apiUrl := "http://example.com" + apiURL := "http://example.com" apiPath := "status" - gock.New(apiUrl). + gock.New(apiURL). Get(apiPath). Reply(302). BodyString("") - _, err := getDataFromURL(apiUrl+"/"+apiPath, ioutil.ReadAll) + instascrap := NewInstascrap(nil) + _, err := instascrap.getDataFromURL(apiURL+"/"+apiPath, ioutil.ReadAll) assert.Error(t, err) } @@ -49,15 +52,16 @@ func TestGetDataFromUrlError(t *testing.T) { func TestGetDataFromUrlNon200HttpCode(t *testing.T) { defer gock.Off() - apiUrl := "http://example.com" + apiURL := "http://example.com" apiPath := "status" - gock.New(apiUrl). + gock.New(apiURL). Get(apiPath). Reply(201). BodyString("") - _, err := getDataFromURL(apiUrl+"/"+apiPath, ioutil.ReadAll) + instascrap := NewInstascrap(nil) + _, err := instascrap.getDataFromURL(apiURL+"/"+apiPath, ioutil.ReadAll) assert.Error(t, err) } @@ -66,15 +70,16 @@ func TestGetDataFromUrlNon200HttpCode(t *testing.T) { func TestGetDataFromUrlBodyReadError(t *testing.T) { defer gock.Off() - apiUrl := "http://example.com" + apiURL := "http://example.com" apiPath := "status" - gock.New(apiUrl). + gock.New(apiURL). Get(apiPath). Reply(200). BodyString("") - _, err := getDataFromURL(apiUrl+"/"+apiPath, func(r io.Reader) ([]byte, error) { + instascrap := NewInstascrap(nil) + _, err := instascrap.getDataFromURL(apiURL+"/"+apiPath, func(r io.Reader) ([]byte, error) { return nil, errors.New("IO Reader error occurred") })